admin/ship-mypy-type-annotations-drop-py37-add-py310-manage-test-dependencies-separate-for-each-tox-env

.github/workflows/build-docs.yml

@@ -14,11 +14,12 @@ jobs:
           fetch-depth: 0
           ref: main
           submodules: true
+
+      # v3 Docs
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
           python-version: 3.9
-
       - name: Checkout v3 Code
         uses: actions/checkout@v2
         with:
@@ -37,6 +38,11 @@ jobs:
         run: |
           rm -Rf .aicsimageio-v3/
 
+      # Latest Docs
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.10'
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

.github/workflows/build-main.yml

@@ -12,19 +12,70 @@ on:
     - cron: "0 18 * * 1"
 
 jobs:
-  test:
+  test-core-lib:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9, '3.10']
         os: [
-          ubuntu-18.04,
           ubuntu-20.04,
           windows-latest,
-          macOS-10.15,
           macos-11,
         ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+      - uses: actions/cache@v2
+        id: cache
+        with:
+          path: aicsimageio/tests/resources
+          key: ${{ hashFiles('scripts/TEST_RESOURCES_HASH.txt') }}
+      - name: Download Test Resources
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python scripts/download_test_resources.py --debug
+      - name: Run tests with Tox
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py
+      - name: Upload codecov
+        uses: codecov/codecov-action@v2
+
+  test-readers:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9, '3.10']
+        os: [
+          ubuntu-20.04,
+          windows-latest,
+          macos-11,
+        ]
+        tox-env: [
+          bioformats,
+          czi,
+          base-imageio,
+          dv,
+          lif,
+          nd2,
+          bfio,
+        ]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -58,9 +109,9 @@ jobs:
           python scripts/download_test_resources.py --debug
       - name: Run tests with Tox
         # Run tox using the version of Python in `PATH`
-        run: tox -e py
+        run: tox -e ${{ matrix.tox-env }}
       - name: Upload codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2
 
   lint:
     runs-on: ubuntu-latest
@@ -71,7 +122,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
@@ -81,7 +132,7 @@ jobs:
 
   publish:
     if: "contains(github.event.head_commit.message, 'Bump version')"
-    needs: [test, lint]
+    needs: [test-core-lib, test-readers, lint]
     runs-on: ubuntu-latest
 
     steps:
@@ -91,7 +142,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/test-and-lint.yml

@@ -2,20 +2,76 @@ name: Test and Lint
 
 on: pull_request
 
+# Cancel actions when new commits are pushed to PR
+concurrency: 
+    group: ${{ github.head_ref }}
+    cancel-in-progress: true
+
 jobs:
-  test:
+  test-core-lib:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9, '3.10']
         os: [
-          ubuntu-18.04,
           ubuntu-20.04,
           windows-latest,
-          macOS-10.15,
           macos-11,
         ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+      - uses: actions/cache@v2
+        id: cache
+        with:
+          path: aicsimageio/tests/resources
+          key: ${{ hashFiles('scripts/TEST_RESOURCES_HASH.txt') }}
+      - name: Download Test Resources
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python scripts/download_test_resources.py --debug
+      - name: Run tests with Tox
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py -- -k "not REMOTE"
+      - name: Upload codecov
+        uses: codecov/codecov-action@v2
+
+  test-readers:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9, '3.10']
+        os: [
+          ubuntu-20.04,
+          windows-latest,
+          macos-11,
+        ]
+        tox-env: [
+          bioformats,
+          czi,
+          base-imageio,
+          dv,
+          lif,
+          nd2,
+          bfio,
+        ]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -28,6 +84,12 @@ jobs:
         with:
           distribution: "temurin"
           java-version: "11"
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
@@ -42,10 +104,9 @@ jobs:
         run: |
           python scripts/download_test_resources.py --debug
       - name: Run tests with Tox
-        # Run tox using the version of Python in `PATH`, and don't include remote resources
-        run: tox -e py -- -k "not REMOTE"
+        run: tox -e ${{ matrix.tox-env }} -- -k "not REMOTE"
       - name: Upload codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2
 
   lint:
     runs-on: ubuntu-latest
@@ -56,7 +117,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
@@ -74,7 +135,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Install Dependencies
         run: |
           pip install --upgrade pip

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       hooks:
           - id: flake8
             additional_dependencies: [flake8-typing-imports, flake8-debugger]
-            args: [--count, --show-source, --statistics, --min-python-version=3.7.0]
+            args: [--count, --show-source, --statistics, --min-python-version=3.8.0]
     - repo: https://github.com/myint/autoflake
       rev: v1.4
       hooks:

MANIFEST.in

@@ -1,5 +1,6 @@
 include LICENSE
 include README.md
+include aicsimageio/py.typed
 
 recursive-include tests *
 recursive-exclude * __pycache__

aicsimageio/readers/bioformats_reader.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import os
-from functools import lru_cache
+from functools import cached_property, lru_cache
 from pathlib import Path
 from threading import Lock
 from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union
@@ -13,12 +13,14 @@
 import xarray as xr
 from fsspec.implementations.local import LocalFileSystem
 from ome_types import OME
+from resource_backed_dask_array import (
+    ResourceBackedDaskArray,
+    resource_backed_dask_array,
+)
 
 from .. import constants, dimensions, exceptions
 from ..metadata import utils as metadata_utils
 from ..utils import io_utils
-from ..utils.cached_property import cached_property
-from ..utils.dask_proxy import DaskArrayProxy
 from .reader import Reader
 
 if TYPE_CHECKING:
@@ -376,21 +378,21 @@ def to_numpy(self, series: Optional[int] = None) -> np.ndarray:
         """
         return np.asarray(self.to_dask(series))
 
-    def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy:
+    def to_dask(self, series: Optional[int] = None) -> ResourceBackedDaskArray:
         """Create dask array for the specified or current series.
 
         Note: the order of the returned array will *always* be `TCZYX[r]`,
         where `[r]` refers to an optional RGB dimension with size 3 or 4.
         If the image is RGB it will have `ndim==6`, otherwise `ndim` will be 5.
 
-        The returned object is a `DaskArrayProxy`, which is a wrapper on a dask array
-        that ensures the file is open when actually reading (computing) a chunk.  It
-        has all the methods and behavior of a dask array.
-        see :class:`aicsimageio.utils.dask_proxy.DaskArrayProxy`.
+        The returned object is a `ResourceBackedDaskArray`, which is a wrapper on
+        a dask array that ensures the file is open when actually reading (computing)
+        a chunk.  It has all the methods and behavior of a dask array.
+        See: https://github.com/tlambert03/resource-backed-dask-array
 
         Returns
         -------
-        DaskArrayProxy
+        ResourceBackedDaskArray
         """
         if series is not None:
             self._r.setSeries(series)
@@ -409,7 +411,7 @@ def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy:
             chunks=chunks,
             dtype=self.core_meta.dtype,
         )
-        return DaskArrayProxy(arr, self)
+        return resource_backed_dask_array(arr, self)
 
     @property
     def closed(self) -> bool:

aicsimageio/readers/czi_reader.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import warnings
 import xml.etree.ElementTree as ET
 from copy import copy
 from pathlib import Path
@@ -97,7 +96,6 @@ def __init__(
         self,
         image: types.PathLike,
         chunk_dims: Union[str, List[str]] = DEFAULT_CHUNK_DIMS,
-        chunk_by_dims: Optional[Union[str, List[str]]] = None,
         include_subblock_metadata: bool = False,
     ):
         # Expand details of provided image
@@ -110,15 +108,6 @@ def __init__(
                 f"Received URI: {self._path}, which points to {type(self._fs)}."
             )
 
-        # Handle deprecated parameter
-        if chunk_by_dims is not None:
-            warnings.warn(
-                "CziReader parameter 'chunk_by_dims' has been renamed to 'chunk_dims'. "
-                "'chunk_by_dims' will be removed in aicsimageio>=4.1.",
-                DeprecationWarning,
-            )
-            chunk_dims = chunk_by_dims
-
         # Store params
         if isinstance(chunk_dims, str):
             chunk_dims = list(chunk_dims)

aicsimageio/readers/dv_reader.py

@@ -3,10 +3,10 @@
 from typing import TYPE_CHECKING, Any, Tuple
 
 from fsspec.implementations.local import LocalFileSystem
+from resource_backed_dask_array import resource_backed_dask_array
 
 from .. import constants, exceptions, types
 from ..utils import io_utils
-from ..utils.dask_proxy import DaskArrayProxy
 from .reader import Reader
 
 if TYPE_CHECKING:
@@ -71,7 +71,7 @@ def _xarr_reformat(self, delayed: bool) -> xr.DataArray:
         with DVFile(self._path) as dv:
             xarr = dv.to_xarray(delayed=delayed, squeeze=False)
             if delayed:
-                xarr.data = DaskArrayProxy(xarr.data, dv)
+                xarr.data = resource_backed_dask_array(xarr.data, dv)
             xarr.attrs[constants.METADATA_UNPROCESSED] = xarr.attrs.pop("metadata")
         return xarr