From 51ef2084e5b451cfde08e6c2de35a50377197e4a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:17:26 -0500
Subject: [PATCH 01/24] Add minimum dependencies CI build

---
 .github/workflows/additional.yml              | 47 +++++++++++++++++++
 .../environment-mindeps.yaml                  | 18 +++++++
 2 files changed, 65 insertions(+)
 create mode 100644 .github/workflows/additional.yml
 create mode 100644 continuous_integration/environment-mindeps.yaml

diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
new file mode 100644
index 00000000000..b89041f0fdc
--- /dev/null
+++ b/.github/workflows/additional.yml
@@ -0,0 +1,47 @@
+name: Additional
+
+on: [push, pull_request]
+
+jobs:
+  mindeps:
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channels: conda-forge,defaults
+          channel-priority: true
+          environment-file: continuous_integration/environment-mindeps.yaml
+          activate-environment: dask-distributed
+          auto-activate-base: false
+
+      - name: mamba list
+        shell: bash -l {0}
+        run: mamba list
+
+      - name: mamba env export
+        shell: bash -l {0}
+        run: |
+          echo -e "--\n--Conda Environment (re-create this with \`mamba env create --name <name> -f <output_file>\`)\n--"
+          mamba env export | grep -E -v '^prefix:.*$'
+
+      - name: Test
+        shell: bash -l {0}
+        env:
+          PYTHONFAULTHANDLER: 1
+        run: |
+          if [[ "${{ matrix.os }}" = "ubuntu-latest" ]]; then
+              # FIXME ipv6-related failures on Ubuntu github actions CI
+              # https://github.com/dask/distributed/issues/4514
+              export DISABLE_IPV6=1
+          fi
+
+          source continuous_integration/scripts/set_ulimit.sh
+          pytest distributed -m "not avoid_ci" --runslow
diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
new file mode 100644
index 00000000000..a26087102c2
--- /dev/null
+++ b/continuous_integration/environment-mindeps.yaml
@@ -0,0 +1,18 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - click=6.6
+  - cloudpickle=1.5.0
+  - dask=2021.03.0
+  - msgpack=0.6.0
+  - psutil=5.0
+  - sortedcontainers=2.0.2
+  - tblib=1.6.0
+  - toolz=0.8.2
+  - tornado=5
+  - zict=0.1.3
+  - pyyaml
+  - setuptools
\ No newline at end of file

From ef68243d84525247f3385a0c899f64176e9cd855 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:27:27 -0500
Subject: [PATCH 02/24] Fix conda solve errors

---
 continuous_integration/environment-mindeps.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index a26087102c2..2c229de4aad 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -7,9 +7,9 @@ dependencies:
   - click=6.6
   - cloudpickle=1.5.0
   - dask=2021.03.0
-  - msgpack=0.6.0
+  - msgpack-python=0.6.0
   - psutil=5.0
-  - sortedcontainers=2.0.2
+  - sortedcontainers=2.0.4
   - tblib=1.6.0
   - toolz=0.8.2
   - tornado=5

From 6eec67f8f71047bd691bd7da762b31d25e307c69 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:33:17 -0500
Subject: [PATCH 03/24] Update minimum psutil

---
 continuous_integration/environment-mindeps.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 2c229de4aad..f7b931d867a 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -8,7 +8,7 @@ dependencies:
   - cloudpickle=1.5.0
   - dask=2021.03.0
   - msgpack-python=0.6.0
-  - psutil=5.0
+  - psutil=5.4.7
   - sortedcontainers=2.0.4
   - tblib=1.6.0
   - toolz=0.8.2

From 88f3c538ab5add17699a6c4e814060f973fc2c3f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:38:41 -0500
Subject: [PATCH 04/24] Specify Python version in workflow file

---
 .github/workflows/additional.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index b89041f0fdc..95188f9ec3f 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -18,6 +18,7 @@ jobs:
           use-mamba: true
           channels: conda-forge,defaults
           channel-priority: true
+          python-version: 3.7
           environment-file: continuous_integration/environment-mindeps.yaml
           activate-environment: dask-distributed
           auto-activate-base: false

From e8716987e57723c09e1fa17e987a4031290e02f1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:45:51 -0500
Subject: [PATCH 05/24] Bump click

---
 continuous_integration/environment-mindeps.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index f7b931d867a..b93c9adef45 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -4,7 +4,7 @@ channels:
   - defaults
 dependencies:
   - python=3.7
-  - click=6.6
+  - click=6.7
   - cloudpickle=1.5.0
   - dask=2021.03.0
   - msgpack-python=0.6.0

From 6d486e02e27f57238c3a893ba1c119ed095da3c2 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:49:24 -0500
Subject: [PATCH 06/24] Add testing deps

---
 continuous_integration/environment-mindeps.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index b93c9adef45..4973606f952 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -15,4 +15,11 @@ dependencies:
   - tornado=5
   - zict=0.1.3
   - pyyaml
-  - setuptools
\ No newline at end of file
+  - setuptools
+  # test dependencies
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
\ No newline at end of file

From a02538aa1b14d89538dfe6d9d65fbfe3eb553d3b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 13:52:51 -0500
Subject: [PATCH 07/24] Make sure distributed is installed

---
 .github/workflows/additional.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index 95188f9ec3f..7fd52162594 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -23,6 +23,10 @@ jobs:
           activate-environment: dask-distributed
           auto-activate-base: false
 
+      - name: Install
+        shell: bash -l {0}
+        run: python -m pip install --no-deps -e .
+
       - name: mamba list
         shell: bash -l {0}
         run: mamba list

From 06cbef562b145a212896e7bbe7ad8d3dc6c0701d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 14:16:55 -0500
Subject: [PATCH 08/24] Update distributed/comm/tests/test_ws.py

---
 distributed/comm/tests/test_ws.py | 40 ++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index a7e305f718a..6f7c6eff1fa 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -22,7 +22,16 @@
 
 from .test_comms import check_tls_extra
 
-security = Security.temporary()
+try:
+    import cryptography
+except ImportError:
+    cryptography = None
+
+
+@pytest.fixture
+def security():
+    pytest.importorskip("cryptography")
+    return Security.temporary()
 
 
 def test_registered():
@@ -77,7 +86,7 @@ async def test_expect_ssl_context(cleanup):
 
 
 @pytest.mark.asyncio
-async def test_expect_scheduler_ssl_when_sharing_server(cleanup):
+async def test_expect_scheduler_ssl_when_sharing_server(cleanup, security):
     with tempfile.TemporaryDirectory() as tempdir:
         key_path = os.path.join(tempdir, "dask.pem")
         cert_path = os.path.join(tempdir, "dask.crt")
@@ -133,16 +142,19 @@ async def test_large_transfer(cleanup):
     "dashboard,protocol,security,port",
     [
         (True, "ws://", None, 8787),
-        (True, "wss://", security, 8787),
+        (True, "wss://", True, 8787),
         (False, "ws://", None, 8787),
-        (False, "wss://", security, 8787),
+        (False, "wss://", True, 8787),
         (True, "ws://", None, 8786),
-        (True, "wss://", security, 8786),
+        (True, "wss://", True, 8786),
         (False, "ws://", None, 8786),
-        (False, "wss://", security, 8786),
+        (False, "wss://", True, 8786),
     ],
 )
 async def test_http_and_comm_server(cleanup, dashboard, protocol, security, port):
+    if security:
+        pytest.importorskip("cryptography")
+        security = Security.temporary()
     async with Scheduler(
         protocol=protocol, dashboard=dashboard, port=port, security=security
     ) as s:
@@ -156,18 +168,27 @@ async def test_http_and_comm_server(cleanup, dashboard, protocol, security, port
                 assert result == 11
 
 
+@pytest.mark.skipif(not cryptography, reason="Requires cryptography")
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "protocol,security",
     [
         (
             "ws://",
-            Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}}),
+            pytest.param(
+                Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}}),
+                marks=pytest.mark.skipif(
+                    not cryptography, reason="Requires cryptography"
+                ),
+            ),
         ),
         (
             "wss://",
-            Security.temporary(
-                extra_conn_args={"headers": {"Authorization": "Token abcd"}}
+            pytest.param(
+                Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}}),
+                marks=pytest.mark.skipif(
+                    not cryptography, reason="Requires cryptography"
+                ),
             ),
         ),
     ],
@@ -203,6 +224,7 @@ async def test_ws_roundtrip(c, s, a, b):
     assert (x == y).all()
 
 
+@pytest.mark.skipif(not cryptography, reason="Requires cryptography")
 @gen_cluster(
     client=True,
     security=security,

From c50f54ca718599c0afedf63069fecb5ddb948d91 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 14:26:10 -0500
Subject: [PATCH 09/24] Make sure DISABLE_IPV6 is set properly

---
 .github/workflows/additional.yml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index 7fd52162594..518e12d2e03 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -41,12 +41,9 @@ jobs:
         shell: bash -l {0}
         env:
           PYTHONFAULTHANDLER: 1
+          # FIXME ipv6-related failures on Ubuntu github actions CI
+          # https://github.com/dask/distributed/issues/4514
+          DISABLE_IPV6: 1
         run: |
-          if [[ "${{ matrix.os }}" = "ubuntu-latest" ]]; then
-              # FIXME ipv6-related failures on Ubuntu github actions CI
-              # https://github.com/dask/distributed/issues/4514
-              export DISABLE_IPV6=1
-          fi
-
           source continuous_integration/scripts/set_ulimit.sh
           pytest distributed -m "not avoid_ci" --runslow

From 761c41980677b9c39ecdeac399bdfb40ea5b1e54 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 16:22:03 -0500
Subject: [PATCH 10/24] Bump minimum Dask version in CI

---
 continuous_integration/environment-mindeps.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 4973606f952..c298d3d3302 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -6,7 +6,7 @@ dependencies:
   - python=3.7
   - click=6.7
   - cloudpickle=1.5.0
-  - dask=2021.03.0
+  - dask=2021.04.1
   - msgpack-python=0.6.0
   - psutil=5.4.7
   - sortedcontainers=2.0.4

From e6c08a37d519ccac913b7e9231c1cf5319731a51 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 3 May 2021 16:47:13 -0500
Subject: [PATCH 11/24] More test_ws.py fixup

---
 distributed/comm/tests/test_ws.py | 67 ++++++++++++-------------------
 1 file changed, 25 insertions(+), 42 deletions(-)

diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index 6f7c6eff1fa..ff08f5d32c6 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -28,12 +28,6 @@
     cryptography = None
 
 
-@pytest.fixture
-def security():
-    pytest.importorskip("cryptography")
-    return Security.temporary()
-
-
 def test_registered():
     assert "ws" in backends
     backend = get_backend("ws")
@@ -86,7 +80,9 @@ async def test_expect_ssl_context(cleanup):
 
 
 @pytest.mark.asyncio
-async def test_expect_scheduler_ssl_when_sharing_server(cleanup, security):
+async def test_expect_scheduler_ssl_when_sharing_server(cleanup):
+    pytest.importorskip("cryptography")
+    security = Security.temporary()
     with tempfile.TemporaryDirectory() as tempdir:
         key_path = os.path.join(tempdir, "dask.pem")
         cert_path = os.path.join(tempdir, "dask.crt")
@@ -168,32 +164,18 @@ async def test_http_and_comm_server(cleanup, dashboard, protocol, security, port
                 assert result == 11
 
 
-@pytest.mark.skipif(not cryptography, reason="Requires cryptography")
 @pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "protocol,security",
-    [
-        (
-            "ws://",
-            pytest.param(
-                Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}}),
-                marks=pytest.mark.skipif(
-                    not cryptography, reason="Requires cryptography"
-                ),
-            ),
-        ),
-        (
-            "wss://",
-            pytest.param(
-                Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}}),
-                marks=pytest.mark.skipif(
-                    not cryptography, reason="Requires cryptography"
-                ),
-            ),
-        ),
-    ],
-)
-async def test_connection_made_with_extra_conn_args(cleanup, protocol, security):
+@pytest.mark.parametrize("protocol", ["ws://", "wss://"])
+async def test_connection_made_with_extra_conn_args(cleanup, protocol):
+    if protocol == "ws://":
+        security = Security(
+            extra_conn_args={"headers": {"Authorization": "Token abcd"}}
+        )
+    else:
+        pytest.importorskip("cryptography")
+        security = Security.temporary(
+            extra_conn_args={"headers": {"Authorization": "Token abcd"}}
+        )
     async with Scheduler(protocol=protocol, security=security) as s:
         connection_args = security.get_connection_args("worker")
         comm = await connect(s.address, **connection_args)
@@ -225,13 +207,14 @@ async def test_ws_roundtrip(c, s, a, b):
 
 
 @pytest.mark.skipif(not cryptography, reason="Requires cryptography")
-@gen_cluster(
-    client=True,
-    security=security,
-    scheduler_kwargs={"protocol": "wss://"},
-)
-async def test_wss_roundtrip(c, s, a, b):
-    x = np.arange(100)
-    future = await c.scatter(x)
-    y = await future
-    assert (x == y).all()
+@pytest.mark.asyncio
+async def test_wss_roundtrip(cleanup):
+    pytest.importorskip("cryptography")
+    security = Security.temporary()
+    async with Scheduler(protocol="wss://", security=security) as s:
+        async with Worker(s.address, security=security) as w:
+            async with Client(s.address, security=security, asynchronous=True) as c:
+                x = np.arange(100)
+                future = await c.scatter(x)
+                y = await future
+                assert (x == y).all()

From 723f549ee93c056275a3110f4cfcedaa3ebd448f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 7 May 2021 11:24:42 -0500
Subject: [PATCH 12/24] Bump minimum version of toolz to avoid related cytoolz
 seg fault

---
 continuous_integration/environment-mindeps.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index c298d3d3302..17e25961a85 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -11,7 +11,7 @@ dependencies:
   - psutil=5.4.7
   - sortedcontainers=2.0.4
   - tblib=1.6.0
-  - toolz=0.8.2
+  - toolz=0.10.0
   - tornado=5
   - zict=0.1.3
   - pyyaml

From 130d51237bb856d8acfc6b4d761963446a86551f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 7 May 2021 12:06:36 -0500
Subject: [PATCH 13/24] Update minimum deps in requirements.txt

---
 requirements.txt | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index fd8159cfefe..8bbcf660538 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,11 @@
-click >= 6.6
+click >= 6.7
 cloudpickle >= 1.5.0
-dask >= 2021.03.0
+dask >= 2021.04.1
 msgpack >= 0.6.0
-psutil >= 5.0
-sortedcontainers !=2.0.0, !=2.0.1
+psutil >= 5.4.7
+sortedcontainers >= 2.0.4
 tblib >= 1.6.0
-toolz >= 0.8.2
+toolz >= 0.10.0
 tornado >= 5;python_version<'3.8'
 tornado >= 6.0.3;python_version>='3.8'
 zict >= 0.1.3

From 3a728abe95a0a3b3f574c71d0023ac1db82ec887 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 7 May 2021 14:02:34 -0500
Subject: [PATCH 14/24] Add fix for test_pickle_empty

---
 distributed/compatibility.py              | 2 ++
 distributed/protocol/tests/test_pickle.py | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 982b7017951..26496b9bcd9 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -1,12 +1,14 @@
 import logging
 import platform
 import sys
+from distutils.version import LooseVersion
 
 import tornado
 
 logging_names = logging._levelToName.copy()
 logging_names.update(logging._nameToLevel)
 
+PY_VERSION = LooseVersion(".".join(map(str, sys.version_info[:3])))
 PYPY = platform.python_implementation().lower() == "pypy"
 MACOS = sys.platform == "darwin"
 WINDOWS = sys.platform.startswith("win")
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 25989bfcde4..56c99897fbb 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -6,11 +6,12 @@
 
 import pytest
 
+from distributed.compatibility import PY_VERSION
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 from distributed.protocol.serialize import pickle_dumps
 
-if sys.version_info < (3, 8):
+if PY_VERSION < "3.8":
     try:
         import pickle5 as pickle
     except ImportError:
@@ -79,7 +80,8 @@ def test_pickle_empty():
     header["writeable"] = [False] * len(frames)
     y = deserialize(header, frames)
     assert memoryview(y).nbytes == 0
-    assert memoryview(y).readonly
+    if HIGHEST_PROTOCOL >= 5:
+        assert memoryview(y).readonly
 
 
 def test_pickle_numpy():

From 5be6f2008190130b4f86c2f7ab844241ac797cbb Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 21 Jun 2021 14:42:15 -0500
Subject: [PATCH 15/24] Latest dask

---
 continuous_integration/environment-mindeps.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 17e25961a85..7412fdf54e1 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -6,7 +6,6 @@ dependencies:
   - python=3.7
   - click=6.7
   - cloudpickle=1.5.0
-  - dask=2021.04.1
   - msgpack-python=0.6.0
   - psutil=5.4.7
   - sortedcontainers=2.0.4
@@ -22,4 +21,7 @@ dependencies:
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
-  - pytest-timeout
\ No newline at end of file
+  - pytest-timeout
+  - pip:
+    # Distributed depends on the latest version of Dask
+    - git+https://github.com/dask/dask
\ No newline at end of file

From 3b4ff445c49ee7c4f46393fc4d141eee88525a02 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 21 Jun 2021 17:17:44 -0500
Subject: [PATCH 16/24] Add pip

---
 continuous_integration/environment-mindeps.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 7412fdf54e1..9f3706e6a4b 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -22,6 +22,7 @@ dependencies:
   - pytest-repeat
   - pytest-rerunfailures
   - pytest-timeout
+  - pip
   - pip:
     # Distributed depends on the latest version of Dask
     - git+https://github.com/dask/dask
\ No newline at end of file

From cd76c2189bff242f48563fba1abb76977a0e5a8a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 21 Jun 2021 17:38:19 -0500
Subject: [PATCH 17/24] Fixup

---
 distributed/comm/tests/test_ws.py                  |  3 ++-
 distributed/protocol/tests/test_collection_cuda.py |  7 +++----
 distributed/protocol/tests/test_highlevelgraph.py  | 10 +++++-----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index 5b1759d51c0..b7e821b865f 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -2,7 +2,6 @@
 import tempfile
 import warnings
 
-import numpy as np
 import pytest
 
 import dask
@@ -199,6 +198,7 @@ async def test_quiet_close(cleanup):
     scheduler_kwargs={"protocol": "ws://"},
 )
 async def test_ws_roundtrip(c, s, a, b):
+    np = pytest.importorskip("numpy")
     x = np.arange(100)
     future = await c.scatter(x)
     y = await future
@@ -209,6 +209,7 @@ async def test_ws_roundtrip(c, s, a, b):
 @pytest.mark.asyncio
 async def test_wss_roundtrip(cleanup):
     pytest.importorskip("cryptography")
+    np = pytest.importorskip("numpy")
     security = Security.temporary()
     async with Scheduler(protocol="wss://", security=security) as s:
         async with Worker(s.address, security=security) as w:
diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py
index a50fb7e2bb8..fd12ad55717 100644
--- a/distributed/protocol/tests/test_collection_cuda.py
+++ b/distributed/protocol/tests/test_collection_cuda.py
@@ -1,7 +1,5 @@
 import pytest
 
-from dask.dataframe.utils import assert_eq
-
 from distributed.protocol import deserialize, serialize
 
 
@@ -42,6 +40,7 @@ def test_serialize_cupy(collection, y, y_serializer):
 def test_serialize_pandas_pandas(collection, df2, df2_serializer):
     cudf = pytest.importorskip("cudf")
     pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
     df1 = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]})
     if df2 is not None:
         df2 = cudf.from_pandas(pd.DataFrame(df2))
@@ -59,8 +58,8 @@ def test_serialize_pandas_pandas(collection, df2, df2_serializer):
     assert sub_headers[1]["serializer"] == df2_serializer
     assert isinstance(t, collection)
 
-    assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1)
+    dd.assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1)
     if df2 is None:
         assert (t["df2"] if isinstance(t, dict) else t[1]) is None
     else:
-        assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2)
+        dd.assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2)
diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py
index df9f2942179..741c0d6c62d 100644
--- a/distributed/protocol/tests/test_highlevelgraph.py
+++ b/distributed/protocol/tests/test_highlevelgraph.py
@@ -2,6 +2,11 @@
 
 import pytest
 
+np = pytest.importorskip("numpy")
+pd = pytest.importorskip("pandas")
+
+from numpy.testing import assert_array_equal
+
 import dask
 import dask.array as da
 import dask.dataframe as dd
@@ -9,11 +14,6 @@
 from distributed.diagnostics import SchedulerPlugin
 from distributed.utils_test import gen_cluster
 
-np = pytest.importorskip("numpy")
-pd = pytest.importorskip("pandas")
-
-from numpy.testing import assert_array_equal
-
 
 @gen_cluster(client=True)
 async def test_combo_of_layer_types(c, s, a, b):

From 1962dddd26c9f6122bd2a2cd1b2d5c94ff518de5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Thu, 18 Nov 2021 17:26:26 -0600
Subject: [PATCH 18/24] Add jinja2

---
 continuous_integration/environment-mindeps.yaml   | 1 +
 distributed/protocol/tests/test_protocol_utils.py | 1 +
 distributed/tests/test_security.py                | 1 +
 3 files changed, 3 insertions(+)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 9f3706e6a4b..0ae14e2c7cd 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -6,6 +6,7 @@ dependencies:
   - python=3.7
   - click=6.7
   - cloudpickle=1.5.0
+  - jinja2
   - msgpack-python=0.6.0
   - psutil=5.4.7
   - sortedcontainers=2.0.4
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index 5ebcb6e1e12..912f041e097 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -66,6 +66,7 @@ def test_readonly_buffer(self):
         assert result == base
 
     def test_catch_non_memoryview(self):
+        pytest.importorskip("numpy")
         with pytest.raises(TypeError, match="Expected memoryview"):
             merge_memoryviews([b"1234", memoryview(b"4567")])
 
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index c9f3c722573..69dab1f3dfb 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -111,6 +111,7 @@ def test_kwargs():
 
 
 def test_repr_temp_keys():
+    pytest.importorskip("cryptography")
     sec = Security.temporary()
     representation = repr(sec)
     assert "Temporary (In-memory)" in representation

From a18f33d655e5dc706114500f1a5ae4f1dc0f4c67 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 7 Dec 2021 16:20:54 -0600
Subject: [PATCH 19/24] msgpack < 1 compatibility

---
 distributed/tests/test_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4209b801446..c30072187b5 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -7113,7 +7113,7 @@ def _verify_cluster_dump(path, _format):
         path += ".msgpack.gz"
 
         with gzip.open(path) as fd:
-            state = msgpack.unpack(fd)
+            state = msgpack.unpack(fd, raw=False)
     else:
         import yaml
 

From 56184abbeb7f5f30dcd03c60d2b9b187d64a2562 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 7 Dec 2021 18:07:42 -0600
Subject: [PATCH 20/24] Possible ssl workaround

---
 continuous_integration/environment-mindeps.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 0ae14e2c7cd..9c69f5a39b1 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -8,6 +8,7 @@ dependencies:
   - cloudpickle=1.5.0
   - jinja2
   - msgpack-python=0.6.0
+  - openssl<3 # TODO: Add note about why this is needed
   - psutil=5.4.7
   - sortedcontainers=2.0.4
   - tblib=1.6.0

From b3fa6cf2912e14bb9e530547148bc5ac0063e94c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Wed, 8 Dec 2021 13:35:32 -0600
Subject: [PATCH 21/24] Add note about openssl version constraint

---
 continuous_integration/environment-mindeps.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 9c69f5a39b1..85b0a14de51 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -8,7 +8,12 @@ dependencies:
   - cloudpickle=1.5.0
   - jinja2
   - msgpack-python=0.6.0
-  - openssl<3 # TODO: Add note about why this is needed
+  # There are known issues when running Python and openssl 3 (see
+  # https://bugs.python.org/issue38820).
+  # Openssl 1.1.1 contains a backport to fix unexpected SSL errors being raised
+  # https://github.com/openssl/openssl/pull/11400.
+  # Make sure openssl < 3 is installed as a temporary workaround.
+  - openssl<3
   - psutil=5.4.7
   - sortedcontainers=2.0.4
   - tblib=1.6.0

From bf0493009307fce91984cf6f04bd5f4b615dfd4d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Thu, 6 Jan 2022 13:16:14 -0600
Subject: [PATCH 22/24] Review feedback

---
 .github/workflows/additional.yml | 45 +++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index 518e12d2e03..92e21c77ae3 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -5,23 +5,28 @@ on: [push, pull_request]
 jobs:
   mindeps:
     runs-on: "ubuntu-latest"
+    timeout-minutes: 180
 
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniforge-variant: Mambaforge
           miniforge-version: latest
+          condarc-file: continuous_integration/condarc
           use-mamba: true
-          channels: conda-forge,defaults
-          channel-priority: true
           python-version: 3.7
           environment-file: continuous_integration/environment-mindeps.yaml
           activate-environment: dask-distributed
-          auto-activate-base: false
+
+      - name: Show conda options
+        shell: bash -l {0}
+        run: conda config --show
 
       - name: Install
         shell: bash -l {0}
@@ -37,6 +42,14 @@ jobs:
           echo -e "--\n--Conda Environment (re-create this with \`mamba env create --name <name> -f <output_file>\`)\n--"
           mamba env export | grep -E -v '^prefix:.*$'
 
+      - name: Setup SSH
+        shell: bash -l {0}
+        run: bash continuous_integration/scripts/setup_ssh.sh
+
+      - name: Reconfigure pytest-timeout
+        shell: bash -l {0}
+        run: sed -i.bak 's/timeout_method = thread/timeout_method = signal/' setup.cfg
+
       - name: Test
         shell: bash -l {0}
         env:
@@ -46,4 +59,28 @@ jobs:
           DISABLE_IPV6: 1
         run: |
           source continuous_integration/scripts/set_ulimit.sh
-          pytest distributed -m "not avoid_ci" --runslow
+          pytest distributed -m "not avoid_ci" --runslow  \
+            --junitxml reports/pytest.xml -o junit_suite_name=mindeps --cov=distributed --cov-report=xml
+
+      - name: Coverage
+        uses: codecov/codecov-action@v1
+
+      - name: Upload test artifacts
+        # ensure this runs even if pytest fails
+        if: >
+          always() &&
+          (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.mindeps }}
+          path: reports
+
+      - name: Upload timeout reports
+        # ensure this runs even if pytest fails
+        if: >
+          always() &&
+          (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.mindeps }}-timeouts
+          path: test_timeout_dump

From 6c549d696d57b8efceff19c5364ecc7c12f89f71 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Thu, 6 Jan 2022 13:24:25 -0600
Subject: [PATCH 23/24] Add pytest-cov to mindeps environment file

---
 continuous_integration/environment-mindeps.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 85b0a14de51..72cb61a87d9 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -25,6 +25,7 @@ dependencies:
   # test dependencies
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures

From d9c39299d7892f7963949f42fcaec832f1e8d174 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Thu, 6 Jan 2022 13:27:21 -0600
Subject: [PATCH 24/24] Update mindeps file

---
 continuous_integration/environment-mindeps.yaml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml
index 72cb61a87d9..9c22933388b 100644
--- a/continuous_integration/environment-mindeps.yaml
+++ b/continuous_integration/environment-mindeps.yaml
@@ -8,12 +8,7 @@ dependencies:
   - cloudpickle=1.5.0
   - jinja2
   - msgpack-python=0.6.0
-  # There are known issues when running Python and openssl 3 (see
-  # https://bugs.python.org/issue38820).
-  # Openssl 1.1.1 contains a backport to fix unexpected SSL errors being raised
-  # https://github.com/openssl/openssl/pull/11400.
-  # Make sure openssl < 3 is installed as a temporary workaround.
-  - openssl<3
+  - packaging=20.0
   - psutil=5.4.7
   - sortedcontainers=2.0.4
   - tblib=1.6.0