diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8ed07abd0..f0e9230c1 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -14,11 +14,12 @@ jobs: fetch-depth: 0 ref: main submodules: true + + # v3 Docs - name: Set up Python uses: actions/setup-python@v1 with: python-version: 3.9 - - name: Checkout v3 Code uses: actions/checkout@v2 with: @@ -37,6 +38,11 @@ jobs: run: | rm -Rf .aicsimageio-v3/ + # Latest Docs + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.10' - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/.github/workflows/build-main.yml b/.github/workflows/build-main.yml index 82106ec0a..b6fab3094 100644 --- a/.github/workflows/build-main.yml +++ b/.github/workflows/build-main.yml @@ -12,19 +12,70 @@ on: - cron: "0 18 * * 1" jobs: - test: + test-core-lib: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, '3.10'] os: [ - ubuntu-18.04, ubuntu-20.04, windows-latest, - macOS-10.15, macos-11, ] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install .[test] + - uses: actions/cache@v2 + id: cache + with: + path: aicsimageio/tests/resources + key: ${{ hashFiles('scripts/TEST_RESOURCES_HASH.txt') }} + - name: Download Test Resources + if: steps.cache.outputs.cache-hit != 'true' + run: | + python scripts/download_test_resources.py --debug + - name: Run tests with Tox + # Run tox using the version of Python in `PATH` + run: tox -e py + - name: Upload codecov + uses: codecov/codecov-action@v2 + + test-readers: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.8, 3.9, '3.10'] + os: [ + ubuntu-20.04, + windows-latest, + macos-11, + ] + tox-env: [ + bioformats, + czi, + base-imageio, + dv, + lif, + nd2, + bfio, + ] steps: - uses: actions/checkout@v2 with: @@ -58,9 +109,9 @@ jobs: python scripts/download_test_resources.py --debug - name: Run tests with Tox # Run tox using the version of Python in `PATH` - run: tox -e py + run: tox -e ${{ matrix.tox-env }} - name: Upload codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 lint: runs-on: ubuntu-latest @@ -71,7 +122,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: '3.10' - name: Install Dependencies run: | python -m pip install --upgrade pip @@ -81,7 +132,7 @@ jobs: publish: if: "contains(github.event.head_commit.message, 'Bump version')" - needs: [test, lint] + needs: [test-core-lib, test-readers, lint] runs-on: ubuntu-latest steps: @@ -91,7 +142,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: '3.10' - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 389e1e334..870defa8d 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -2,20 +2,76 @@ name: Test and Lint on: pull_request +# Cancel actions when new commits are pushed to PR +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true + jobs: - test: + test-core-lib: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, '3.10'] os: [ - ubuntu-18.04, ubuntu-20.04, windows-latest, - macOS-10.15, macos-11, ] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install .[test] + - uses: actions/cache@v2 + id: cache + with: + path: aicsimageio/tests/resources + key: ${{ hashFiles('scripts/TEST_RESOURCES_HASH.txt') }} + - name: Download Test Resources + if: steps.cache.outputs.cache-hit != 'true' + run: | + python scripts/download_test_resources.py --debug + - name: Run tests with Tox + # Run tox using the version of Python in `PATH` + run: tox -e py -- -k "not REMOTE" + - name: Upload codecov + uses: codecov/codecov-action@v2 + + test-readers: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.8, 3.9, '3.10'] + os: [ + ubuntu-20.04, + windows-latest, + macos-11, + ] + tox-env: [ + bioformats, + czi, + base-imageio, + dv, + lif, + nd2, + bfio, + ] steps: - uses: actions/checkout@v2 with: @@ -28,6 +84,12 @@ jobs: with: distribution: "temurin" java-version: "11" + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 - name: Install Dependencies run: | python -m pip install --upgrade pip @@ -42,10 +104,9 @@ jobs: run: | python scripts/download_test_resources.py --debug - name: Run tests with Tox - # Run tox using the version of Python in `PATH`, and don't include remote resources - run: tox -e py -- -k "not REMOTE" + run: tox -e ${{ matrix.tox-env }} -- -k "not REMOTE" - name: Upload codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 lint: runs-on: ubuntu-latest @@ -56,7 +117,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: '3.10' - name: Install Dependencies run: | python -m pip install --upgrade pip @@ -74,7 +135,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: '3.10' - name: Install Dependencies run: | pip install --upgrade pip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab5988378..6eb0c0658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: hooks: - id: flake8 additional_dependencies: [flake8-typing-imports, flake8-debugger] - args: [--count, --show-source, --statistics, --min-python-version=3.7.0] + args: [--count, --show-source, --statistics, --min-python-version=3.8.0] - repo: https://github.com/myint/autoflake rev: v1.4 hooks: diff --git a/MANIFEST.in b/MANIFEST.in index b1b8f1718..3a387f225 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include LICENSE include README.md +include aicsimageio/py.typed recursive-include tests * recursive-exclude * __pycache__ diff --git a/aicsimageio/py.typed b/aicsimageio/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index d0f72f863..18a082ed4 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -3,7 +3,7 @@ from __future__ import annotations import os -from functools import lru_cache +from functools import cached_property, lru_cache from pathlib import Path from threading import Lock from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union @@ -13,12 +13,14 @@ import xarray as xr from fsspec.implementations.local import LocalFileSystem from ome_types import OME +from resource_backed_dask_array import ( + ResourceBackedDaskArray, + resource_backed_dask_array, +) from .. import constants, dimensions, exceptions from ..metadata import utils as metadata_utils from ..utils import io_utils -from ..utils.cached_property import cached_property -from ..utils.dask_proxy import DaskArrayProxy from .reader import Reader if TYPE_CHECKING: @@ -376,21 +378,21 @@ def to_numpy(self, series: Optional[int] = None) -> np.ndarray: """ return np.asarray(self.to_dask(series)) - def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: + def to_dask(self, series: Optional[int] = None) -> ResourceBackedDaskArray: """Create dask array for the specified or current series. Note: the order of the returned array will *always* be `TCZYX[r]`, where `[r]` refers to an optional RGB dimension with size 3 or 4. If the image is RGB it will have `ndim==6`, otherwise `ndim` will be 5. - The returned object is a `DaskArrayProxy`, which is a wrapper on a dask array - that ensures the file is open when actually reading (computing) a chunk. It - has all the methods and behavior of a dask array. - see :class:`aicsimageio.utils.dask_proxy.DaskArrayProxy`. + The returned object is a `ResourceBackedDaskArray`, which is a wrapper on + a dask array that ensures the file is open when actually reading (computing) + a chunk. It has all the methods and behavior of a dask array. + See: https://github.com/tlambert03/resource-backed-dask-array Returns ------- - DaskArrayProxy + ResourceBackedDaskArray """ if series is not None: self._r.setSeries(series) @@ -409,7 +411,7 @@ def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: chunks=chunks, dtype=self.core_meta.dtype, ) - return DaskArrayProxy(arr, self) + return resource_backed_dask_array(arr, self) @property def closed(self) -> bool: diff --git a/aicsimageio/readers/czi_reader.py b/aicsimageio/readers/czi_reader.py index 1722fed2c..e4383f597 100644 --- a/aicsimageio/readers/czi_reader.py +++ b/aicsimageio/readers/czi_reader.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import warnings import xml.etree.ElementTree as ET from copy import copy from pathlib import Path @@ -97,7 +96,6 @@ def __init__( self, image: types.PathLike, chunk_dims: Union[str, List[str]] = DEFAULT_CHUNK_DIMS, - chunk_by_dims: Optional[Union[str, List[str]]] = None, include_subblock_metadata: bool = False, ): # Expand details of provided image @@ -110,15 +108,6 @@ def __init__( f"Received URI: {self._path}, which points to {type(self._fs)}." ) - # Handle deprecated parameter - if chunk_by_dims is not None: - warnings.warn( - "CziReader parameter 'chunk_by_dims' has been renamed to 'chunk_dims'. " - "'chunk_by_dims' will be removed in aicsimageio>=4.1.", - DeprecationWarning, - ) - chunk_dims = chunk_by_dims - # Store params if isinstance(chunk_dims, str): chunk_dims = list(chunk_dims) diff --git a/aicsimageio/readers/dv_reader.py b/aicsimageio/readers/dv_reader.py index 360fcacd4..f7bc5e765 100644 --- a/aicsimageio/readers/dv_reader.py +++ b/aicsimageio/readers/dv_reader.py @@ -3,10 +3,10 @@ from typing import TYPE_CHECKING, Any, Tuple from fsspec.implementations.local import LocalFileSystem +from resource_backed_dask_array import resource_backed_dask_array from .. import constants, exceptions, types from ..utils import io_utils -from ..utils.dask_proxy import DaskArrayProxy from .reader import Reader if TYPE_CHECKING: @@ -71,7 +71,7 @@ def _xarr_reformat(self, delayed: bool) -> xr.DataArray: with DVFile(self._path) as dv: xarr = dv.to_xarray(delayed=delayed, squeeze=False) if delayed: - xarr.data = DaskArrayProxy(xarr.data, dv) + xarr.data = resource_backed_dask_array(xarr.data, dv) xarr.attrs[constants.METADATA_UNPROCESSED] = xarr.attrs.pop("metadata") return xarr diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index f12f7b53a..a816c7c34 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -31,7 +31,6 @@ class TiffGlobReader(Reader): - r""" Wraps the tifffile imread API to provide the same aicsimageio Reader API but for multifile tiff datasets (and other tifffile supported) images. @@ -243,12 +242,7 @@ def indexer(x: str) -> pd.Series: if single_file_shape is None: with self._fs.open(self._path) as open_resource: with TiffFile(open_resource) as tiff: - if tiff.is_shaped: - self._single_file_shape = tuple( - tiff.shaped_metadata[0]["shape"] - ) - elif len(tiff.series) == 1: - self._single_file_shape = tiff.series[0].shape + self._single_file_shape = tiff.series[0].shape else: self._single_file_shape = single_file_shape diff --git a/aicsimageio/tests/metadata/test_utils.py b/aicsimageio/tests/metadata/test_utils.py deleted file mode 100644 index e87832711..000000000 --- a/aicsimageio/tests/metadata/test_utils.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import pytest -from ome_types import OME - -from aicsimageio import AICSImage - -from ..conftest import LOCAL, get_resource_full_path - -############################################################################### - - -@pytest.mark.parametrize( - "filename", - [ - # DefaultReader - pytest.param( - "example.png", - marks=pytest.mark.raises(exception=NotImplementedError), - ), - # TiffReader - pytest.param( - "s_1_t_10_c_3_z_1.tiff", - marks=pytest.mark.raises(exception=NotImplementedError), - ), - # OmeTiffReader - "actk.ome.tiff", - # LifReader - pytest.param( - "tiled.lif", - marks=pytest.mark.raises(exception=NotImplementedError), - ), - # CziReader - "s_1_t_1_c_1_z_1.czi", - "RGB-8bit.czi", - ], -) -def test_ome_metadata(filename: str) -> None: - # Get full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Init image - img = AICSImage(uri) - - # Test the transform - assert isinstance(img.ome_metadata, OME) diff --git a/aicsimageio/tests/readers/extra_readers/__init__.py b/aicsimageio/tests/readers/extra_readers/__init__.py new file mode 100644 index 000000000..faa18be5b --- /dev/null +++ b/aicsimageio/tests/readers/extra_readers/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- diff --git a/aicsimageio/tests/readers/test_bioformats_reader.py b/aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py similarity index 84% rename from aicsimageio/tests/readers/test_bioformats_reader.py rename to aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py index 5a4a07ef0..65a46823a 100644 --- a/aicsimageio/tests/readers/test_bioformats_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py @@ -1,20 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List, Tuple +from typing import Any, List, Tuple, Union import numpy as np import pytest from ome_types import OME -from aicsimageio import dimensions, exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers.bioformats_reader import BioFile, BioformatsReader from aicsimageio.tests.image_container_test_utils import ( run_image_file_checks, run_multi_scene_image_read_checks, ) -from ..conftest import LOCAL, get_resource_full_path, host +from ...conftest import LOCAL, get_resource_full_path, host @host @@ -407,7 +407,7 @@ def test_multi_scene_bioformats_reader( def test_biofile_scene_change() -> None: - """Make sure that DaskArrayProxy doesn't close an opened file.""" + """Make sure that ResourceBackedDaskArray doesn't close an opened file.""" uri = get_resource_full_path("ND2_dims_p4z5t3c2y32x32.nd2", LOCAL) f = BioFile(uri) assert isinstance(f.to_dask().compute(), np.ndarray) @@ -460,3 +460,79 @@ def test_bioformats_dask_tiling_read(filename: str) -> None: np.testing.assert_array_equal(arr_tiled, arr_fullplane) np.testing.assert_array_equal(arr_tiled_set, arr_fullplane) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "Olympus-OIR_etienne_amy_slice_z_stack_0001.oir", + "Olympus-OIR_etienne_amy_slice_z_stack_0001.oir", + ("Olympus-OIR_etienne_amy_slice_z_stack_0001.oir",), + (32, 1, 1, 512, 512), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["CH3"], + (1.0, 1.242961138804478, 1.242961138804478), + OME, + ), + ( + "Imaris-IMS_davemason_Convallaria_3C_1T_confocal.ims", + "Imaris-IMS_davemason_Convallaria_3C_1T_confocal.ims Resolution Level 1", + ("Imaris-IMS_davemason_Convallaria_3C_1T_confocal.ims Resolution Level 1",), + (1, 3, 1, 1024, 1024), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2"], + (0.001, 1.2059374999999999, 1.2059570312500014), + OME, + ), + ( + "DICOM_samples_MR-MONO2-8-16x-heart.dcm", + "Series 0", + ("Series 0",), + (1, 1, 16, 256, 256), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + (None, None, None), + OME, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) diff --git a/aicsimageio/tests/readers/test_czi_reader.py b/aicsimageio/tests/readers/extra_readers/test_czi_reader.py similarity index 59% rename from aicsimageio/tests/readers/test_czi_reader.py rename to aicsimageio/tests/readers/extra_readers/test_czi_reader.py index bdc3ae375..5013d700e 100644 --- a/aicsimageio/tests/readers/test_czi_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_czi_reader.py @@ -2,19 +2,23 @@ # -*- coding: utf-8 -*- import xml.etree.ElementTree as ET -from typing import List, Tuple +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, List, Optional, Tuple, Union import numpy as np import pytest +from ome_types import OME -from aicsimageio import dimensions, exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers import ArrayLikeReader from aicsimageio.readers.czi_reader import CziReader -from ..conftest import LOCAL, REMOTE, get_resource_full_path -from ..image_container_test_utils import ( +from ...conftest import LOCAL, REMOTE, get_resource_full_path +from ...image_container_test_utils import ( run_image_container_mosaic_checks, run_image_file_checks, + run_multi_scene_image_read_checks, ) @@ -406,3 +410,273 @@ def test_czi_reader_mosaic_coords( reader.mosaic_xarray_dask_data.coords[dimensions.DimensionNames.SpatialX].data, expected_mosaic_x_coords, ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "s_1_t_1_c_1_z_1.czi", + "Image:0", + ("Image:0",), + (1, 1, 1, 325, 475), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Bright"], + (None, 1.0833333333333333, 1.0833333333333333), + ET.Element, + ), + ( + "RGB-8bit.czi", + "Image:0", + ("Image:0",), + (1, 1, 1, 624, 924, 3), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0"], + (None, 1.0833333333333333, 1.0833333333333333), + ET.Element, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) + + +@pytest.mark.parametrize( + "filename, " + "first_scene_id, " + "first_scene_shape, " + "second_scene_id, " + "second_scene_shape", + [ + ( + "s_3_t_1_c_3_z_5.czi", + "P2", + (1, 3, 5, 325, 475), + "P3", + (1, 3, 5, 325, 475), + ), + ( + "s_3_t_1_c_3_z_5.czi", + "P3", + (1, 3, 5, 325, 475), + "P1", + (1, 3, 5, 325, 475), + ), + ( + "s_3_t_1_c_3_z_5.czi", + 1, + (1, 3, 5, 325, 475), + 2, + (1, 3, 5, 325, 475), + ), + pytest.param( + "s_3_t_1_c_3_z_5.czi", + ["this is not a scene id"], + None, + None, + None, + marks=pytest.mark.raises(exception=TypeError), + ), + ], +) +def test_multi_scene_aicsimage( + filename: str, + first_scene_id: str, + first_scene_shape: Tuple[int, ...], + second_scene_id: str, + second_scene_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_multi_scene_image_read_checks( + ImageContainer=AICSImage, + image=uri, + first_scene_id=first_scene_id, + first_scene_shape=first_scene_shape, + first_scene_dtype=np.dtype(np.uint16), + second_scene_id=second_scene_id, + second_scene_shape=second_scene_shape, + second_scene_dtype=np.dtype(np.uint16), + ) + + +@pytest.mark.parametrize( + "filename, expected_shape", + [ + ("RGB-8bit.czi", (1, 1, 1, 624, 924, 3)), + ], +) +def test_no_scene_prop_access( + filename: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Construct image and check no scene call with property access + img = AICSImage(uri) + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename, select_scenes", + [ + ("s_1_t_1_c_1_z_1.czi", None), + ("s_3_t_1_c_3_z_5.czi", ["P2", "P1"]), + ], +) +def test_roundtrip_save_all_scenes( + filename: str, select_scenes: Optional[List[str]] +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Read initial + original = AICSImage(uri) + + # Save to temp and compare + with TemporaryDirectory() as tmpdir: + save_path = Path(tmpdir) / f"converted-{filename}.ome.tiff" + original.save(save_path, select_scenes=select_scenes) + + # Re-read + result = AICSImage(save_path) + + # Compare all scenes + # They may not have the same scene ids as some readers use scene names as the + # id, see LifReader for example + if select_scenes is None: + select_original_scenes = list(original.scenes) + else: + select_original_scenes = select_scenes + + assert len(select_original_scenes) == len(result.scenes) + for original_scene_id, result_scene_id in zip( + select_original_scenes, result.scenes + ): + # Compare + original.set_scene(original_scene_id) + result.set_scene(result_scene_id) + + np.testing.assert_array_equal(original.data, result.data) + assert original.dims.order == result.dims.order + assert original.channel_names == result.channel_names + + +@pytest.mark.parametrize( + "filename, " + "reconstruct_mosaic, " + "set_scene, " + "expected_shape, " + "expected_mosaic_tile_dims, " + "specific_tile_index", + [ + ( + "OverViewScan.czi", + True, + "TR1", + (1, 1, 1, 3212, 7398), + (440, 544), + 0, + ), + ( + "OverViewScan.czi", + False, + "TR1", + (120, 1, 1, 1, 440, 544), + (440, 544), + 0, + ), + pytest.param( + "actk.ome.tiff", + True, + "Image:0", + (1, 6, 65, 233, 345), + None, + 0, + # AttributeError raises not because of error in rollback + # but because cannot access Y or X from + # None return from `mosaic_tile_dims` because + # image is not a mosaic tiled image + marks=pytest.mark.raises(exception=AttributeError), + ), + ], +) +def test_mosaic_passthrough( + filename: str, + reconstruct_mosaic: bool, + set_scene: str, + expected_shape: Tuple[int, ...], + expected_mosaic_tile_dims: Tuple[int, int], + specific_tile_index: int, +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, reconstruct_mosaic=reconstruct_mosaic) + img.set_scene(set_scene) + + # Assert basics + assert img.shape == expected_shape + assert img.mosaic_tile_dims.Y == expected_mosaic_tile_dims[0] # type: ignore + assert img.mosaic_tile_dims.X == expected_mosaic_tile_dims[1] # type: ignore + + # Ensure that regardless of stitched or not, we can get tile position + img.get_mosaic_tile_position(specific_tile_index) + + +@pytest.mark.parametrize( + "filename", + [ + "s_1_t_1_c_1_z_1.czi", + "RGB-8bit.czi", + ], +) +def test_ome_metadata(filename: str) -> None: + # Get full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init image + img = AICSImage(uri) + + # Test the transform + assert isinstance(img.ome_metadata, OME) diff --git a/aicsimageio/tests/readers/extra_readers/test_default_reader.py b/aicsimageio/tests/readers/extra_readers/test_default_reader.py new file mode 100644 index 000000000..8a37096c6 --- /dev/null +++ b/aicsimageio/tests/readers/extra_readers/test_default_reader.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +import numpy as np +import pytest + +from aicsimageio import AICSImage, dimensions, exceptions +from aicsimageio.readers.default_reader import DefaultReader +from aicsimageio.readers.reader import Reader + +from ...conftest import LOCAL, get_resource_full_path, host +from ...image_container_test_utils import run_image_file_checks + + +@host +@pytest.mark.parametrize( + "filename, set_scene, expected_shape, expected_dims_order", + [ + ("example.bmp", "Image:0", (480, 640, 4), "YXS"), + ("example.png", "Image:0", (800, 537, 4), "YXS"), + ("example.jpg", "Image:0", (452, 400, 3), "YXS"), + ("example.gif", "Image:0", (72, 268, 268, 4), "TYXS"), + ( + "example_invalid_frame_count.mp4", + "Image:0", + (55, 1080, 1920, 3), + "TYXS", + ), + ( + "example_valid_frame_count.mp4", + "Image:0", + (72, 272, 272, 3), + "TYXS", + ), + pytest.param( + "example.txt", + None, + None, + None, + marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError), + ), + pytest.param( + "example.png", + "Image:1", + None, + None, + marks=pytest.mark.raises(exception=IndexError), + ), + ], +) +def test_default_reader( + filename: str, + host: str, + set_scene: str, + expected_shape: Tuple[int, ...], + expected_dims_order: str, +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, host) + + # Run checks + run_image_file_checks( + ImageContainer=DefaultReader, + image=uri, + set_scene=set_scene, + expected_scenes=("Image:0",), + expected_current_scene="Image:0", + expected_shape=expected_shape, + expected_dtype=np.dtype(np.uint8), + expected_dims_order=expected_dims_order, + expected_channel_names=None, + expected_physical_pixel_sizes=(None, None, None), + expected_metadata_type=dict, + ) + + +def test_ffmpeg_header_fail() -> None: + with pytest.raises(IOError): + # Big Buck Bunny + DefaultReader("https://archive.org/embed/archive-video-files/test.mp4") + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "example.png", + "Image:0", + ("Image:0",), + (1, 1, 1, 800, 537, 4), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0"], + (None, None, None), + dict, + ), + ( + "example.gif", + "Image:0", + ("Image:0",), + (72, 1, 1, 268, 268, 4), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0"], + (None, None, None), + dict, + ), + ( + "example_valid_frame_count.mp4", + "Image:0", + ("Image:0",), + (72, 1, 1, 272, 272, 3), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0"], + (None, None, None), + dict, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "set_dims, " + "set_channel_names, " + "expected_dims, " + "expected_channel_names, " + "expected_shape", + [ + # First check to show nothing changes + ( + "example.gif", + "Image:0", + None, + None, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0"], + (72, 1, 1, 268, 268, 4), + ), + # Check just dims to see default channel name creation + ( + "example.gif", + "Image:0", + "ZYXC", + None, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2", "Channel:0:3"], + (1, 4, 72, 268, 268), + ), + # Check setting both as simple definitions + ( + "example.gif", + "Image:0", + "ZYXC", + ["Red", "Green", "Blue", "Alpha"], + dimensions.DEFAULT_DIMENSION_ORDER, + ["Red", "Green", "Blue", "Alpha"], + (1, 4, 72, 268, 268), + ), + # Check providing too many dims + pytest.param( + "example.gif", + "Image:0", + "ABCDEFG", + None, + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + # Check providing too many channels + pytest.param( + "example.gif", + "Image:0", + "ZYXC", + ["A", "B", "C"], + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + # Check providing channels but no channel dim + pytest.param( + "example.gif", + "Image:0", + None, + ["A", "B", "C"], + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + ], +) +def test_set_coords( + filename: str, + set_scene: str, + set_dims: Optional[Union[str, List[str]]], + set_channel_names: Optional[Union[List[str], List[List[str]]]], + expected_dims: str, + expected_channel_names: List[str], + expected_shape: Tuple[int, ...], +) -> None: + # As a reminder, AICSImage always has certain dimensions + # If you provide a dimension that isn't one of those, + # it will only be available on the reader, not the AICSImage object. + + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, dim_order=set_dims, channel_names=set_channel_names) + + # Set scene + img.set_scene(set_scene) + + # Compare AICSImage results + assert img.dims.order == expected_dims + assert img.channel_names == expected_channel_names + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename, set_reader, extra_kwargs, expected_dims, expected_shape", + [ + # See shape to see why you should use TiffReader :) + ( + "actk.ome.tiff", + DefaultReader, + {}, + dimensions.DEFAULT_DIMENSION_ORDER, + (390, 1, 1, 233, 345), + ), + ], +) +def test_set_reader( + filename: str, + set_reader: Type[Reader], + extra_kwargs: Dict[str, Any], + expected_dims: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, reader=set_reader, **extra_kwargs) + + # Assert basics + assert img.dims.order == expected_dims + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename, expected_shape", + [ + ("example.png", (1, 1, 1, 800, 537, 4)), + ], +) +def test_no_scene_prop_access( + filename: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Construct image and check no scene call with property access + img = AICSImage(uri) + assert img.shape == expected_shape diff --git a/aicsimageio/tests/readers/test_dv_reader.py b/aicsimageio/tests/readers/extra_readers/test_dv_reader.py similarity index 56% rename from aicsimageio/tests/readers/test_dv_reader.py rename to aicsimageio/tests/readers/extra_readers/test_dv_reader.py index a4ae1ee76..7fa865b69 100644 --- a/aicsimageio/tests/readers/test_dv_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_dv_reader.py @@ -1,16 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List, Tuple +from typing import Any, List, Tuple, Union import numpy as np import pytest -from aicsimageio import dimensions, exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers import DVReader from aicsimageio.tests.image_container_test_utils import run_image_file_checks -from ..conftest import get_resource_full_path, host +from ...conftest import LOCAL, get_resource_full_path, host @host @@ -85,3 +85,57 @@ def test_dv_reader( expected_physical_pixel_sizes=expected_physical_pixel_sizes, expected_metadata_type=dict, ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "DV_siRNAi-HeLa_IN_02.r3d_D3D.dv", + "Image:0", + ("Image:0",), + (1, 4, 40, 512, 512), + np.int16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["360/457", "490/528", "555/617", "640/685"], + (0.20000000298023224, 0.06502940505743027, 0.06502940505743027), + dict, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) diff --git a/aicsimageio/tests/readers/test_lif_reader.py b/aicsimageio/tests/readers/extra_readers/test_lif_reader.py similarity index 58% rename from aicsimageio/tests/readers/test_lif_reader.py rename to aicsimageio/tests/readers/extra_readers/test_lif_reader.py index 6ee51740e..a1462c7a1 100644 --- a/aicsimageio/tests/readers/test_lif_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_lif_reader.py @@ -1,18 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import time import xml.etree.ElementTree as ET -from typing import List, Optional, Tuple +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import pytest +from distributed import Client, LocalCluster from readlif.reader import LifFile -from aicsimageio import dimensions, exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers.lif_reader import LifReader -from ..conftest import LOCAL, get_resource_full_path, host -from ..image_container_test_utils import ( +from ...conftest import LOCAL, get_resource_full_path, host +from ...image_container_test_utils import ( run_image_container_mosaic_checks, run_image_file_checks, ) @@ -345,3 +349,245 @@ def test_lif_reader_mosaic_coords( reader.mosaic_xarray_dask_data.coords[dimensions.DimensionNames.SpatialX].data, expected_mosaic_x_coords, ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "s_1_t_4_c_2_z_1.lif", + "b2_001_Crop001_Resize001", + ("b2_001_Crop001_Resize001",), + (4, 2, 1, 614, 614), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Gray--TL-PH--EMP_BF", "Green--FLUO--GFP"], + (None, 0.33914910277324634, 0.33914910277324634), + ET.Element, + ), + ( + "tiled.lif", + "TileScan_002", + ("TileScan_002",), + (1, 4, 1, 5622, 7666), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Gray", "Red", "Green", "Cyan"], + (None, 0.20061311154598827, 0.20061311154598827), + ET.Element, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) + + +@pytest.mark.parametrize( + "filename, select_scenes", + [ + ("s_1_t_4_c_2_z_1.lif", None), + ("tiled.lif", None), + ], +) +def test_roundtrip_save_all_scenes( + filename: str, select_scenes: Optional[List[str]] +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Read initial + original = AICSImage(uri) + + # Save to temp and compare + with TemporaryDirectory() as tmpdir: + save_path = Path(tmpdir) / f"converted-{filename}.ome.tiff" + original.save(save_path, select_scenes=select_scenes) + + # Re-read + result = AICSImage(save_path) + + # Compare all scenes + # They may not have the same scene ids as some readers use scene names as the + # id, see LifReader for example + if select_scenes is None: + select_original_scenes = list(original.scenes) + else: + select_original_scenes = select_scenes + + assert len(select_original_scenes) == len(result.scenes) + for original_scene_id, result_scene_id in zip( + select_original_scenes, result.scenes + ): + # Compare + original.set_scene(original_scene_id) + result.set_scene(result_scene_id) + + np.testing.assert_array_equal(original.data, result.data) + assert original.dims.order == result.dims.order + assert original.channel_names == result.channel_names + + +@pytest.mark.parametrize( + "filename, " + "reconstruct_mosaic, " + "set_scene, " + "expected_shape, " + "expected_mosaic_tile_dims, " + "specific_tile_index", + [ + ( + "tiled.lif", + True, + "TileScan_002", + (1, 4, 1, 5622, 7666), + (512, 512), + 0, + ), + ( + "tiled.lif", + False, + "TileScan_002", + (165, 1, 4, 1, 512, 512), + (512, 512), + 0, + ), + pytest.param( + "tiled.lif", + False, + "TileScan_002", + (165, 1, 4, 1, 512, 512), + (512, 512), + 999, + marks=pytest.mark.raises(exception=IndexError), + ), + ], +) +def test_mosaic_passthrough( + filename: str, + reconstruct_mosaic: bool, + set_scene: str, + expected_shape: Tuple[int, ...], + expected_mosaic_tile_dims: Tuple[int, int], + specific_tile_index: int, +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, reconstruct_mosaic=reconstruct_mosaic) + img.set_scene(set_scene) + + # Assert basics + assert img.shape == expected_shape + assert img.mosaic_tile_dims.Y == expected_mosaic_tile_dims[0] # type: ignore + assert img.mosaic_tile_dims.X == expected_mosaic_tile_dims[1] # type: ignore + + # Ensure that regardless of stitched or not, we can get tile position + img.get_mosaic_tile_position(specific_tile_index) + + +@pytest.mark.parametrize( + "filename, set_scene, get_dims, get_specific_dims, expected_shape", + [ + ( + "s_1_t_4_c_2_z_1.lif", + "b2_001_Crop001_Resize001", + "TYX", + {}, + (4, 614, 614), + ), + # Check mosaic chunk handling + ( + "tiled.lif", + "TileScan_002", + "CYX", + {"Y": slice(0, 2000), "X": slice(0, 2000)}, + (4, 2000, 2000), + ), + ], +) +@pytest.mark.parametrize("chunk_dims", ["YX", "ZYX"]) +@pytest.mark.parametrize("processes", [True, False]) +def test_parallel_read( + filename: str, + set_scene: str, + chunk_dims: str, + processes: bool, + get_dims: str, + get_specific_dims: Dict[str, Union[int, slice, range, Tuple[int, ...], List[int]]], + expected_shape: Tuple[int, ...], +) -> None: + """ + This test ensures that our produced dask array can be read in parallel. + """ + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init image + img = AICSImage(uri, chunk_dims=chunk_dims) + img.set_scene(set_scene) + + # Init cluster + cluster = LocalCluster(processes=processes) + client = Client(cluster) + + # Select data + out = img.get_image_dask_data(get_dims, **get_specific_dims).compute() + assert out.shape == expected_shape + + # Shutdown and then safety measure timeout + cluster.close() + client.close() + time.sleep(5) + + +@pytest.mark.parametrize( + "filename, expected_shape", + [ + ("s_1_t_4_c_2_z_1.lif", (4, 2, 1, 614, 614)), + ], +) +def test_no_scene_prop_access( + filename: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Construct image and check no scene call with property access + img = AICSImage(uri) + assert img.shape == expected_shape diff --git a/aicsimageio/tests/readers/test_nd2_reader.py b/aicsimageio/tests/readers/extra_readers/test_nd2_reader.py similarity index 72% rename from aicsimageio/tests/readers/test_nd2_reader.py rename to aicsimageio/tests/readers/extra_readers/test_nd2_reader.py index def9448e8..dd38a0e22 100644 --- a/aicsimageio/tests/readers/test_nd2_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_nd2_reader.py @@ -1,16 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List, Tuple +from typing import Any, List, Tuple, Union import numpy as np import pytest -from aicsimageio import exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers.nd2_reader import ND2Reader from aicsimageio.tests.image_container_test_utils import run_image_file_checks -from ..conftest import get_resource_full_path, host +from ...conftest import LOCAL, get_resource_full_path, host @host @@ -165,3 +165,57 @@ def test_nd2_reader( expected_physical_pixel_sizes=expected_physical_pixel_sizes, expected_metadata_type=dict, ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "ND2_jonas_header_test2.nd2", + "XYPos:0", + ("XYPos:0",), + (4, 1, 5, 520, 696), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Jonas_DIC"], + (0.5, 0.12863494437945, 0.12863494437945), + dict, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) diff --git a/aicsimageio/tests/readers/test_ome_tiled_tiff_reader.py b/aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py similarity index 84% rename from aicsimageio/tests/readers/test_ome_tiled_tiff_reader.py rename to aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py index 0c76782fd..3359e877f 100644 --- a/aicsimageio/tests/readers/test_ome_tiled_tiff_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py @@ -1,17 +1,18 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List, Tuple +from typing import List, Tuple, Type import numpy as np import pytest from ome_types import OME -from aicsimageio import dimensions, exceptions +from aicsimageio import AICSImage, dimensions, exceptions, readers from aicsimageio.readers.bfio_reader import OmeTiledTiffReader +from aicsimageio.readers.reader import Reader -from ..conftest import LOCAL, get_resource_full_path, host -from ..image_container_test_utils import run_image_file_checks +from ...conftest import LOCAL, get_resource_full_path, host +from ...image_container_test_utils import run_image_file_checks @host @@ -208,3 +209,38 @@ def test_ome_tiff_reader_large_files( expected_physical_pixel_sizes=expected_physical_pixel_sizes, expected_metadata_type=OME, ) + + +@pytest.mark.parametrize( + "filename, expected_reader", + [ + ( + "s_1_t_1_c_1_z_1.tiff", + readers.TiffReader, + ), + ( + "s_1_t_1_c_1_z_1.ome.tiff", + readers.OmeTiffReader, + ), + ( + "s_1_t_1_c_1_z_1_ome_tiff_tiles.ome.tif", + readers.OmeTiledTiffReader, + ), + ( # Multiscene tiff + "variable_scene_shape_first_scene_pyramid.ome.tiff", + readers.OmeTiffReader, + ), + ], +) +def test_selected_tiff_reader( + filename: str, + expected_reader: Type[Reader], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri) + + # Assert basics + assert isinstance(img.reader, expected_reader) diff --git a/aicsimageio/tests/readers/test_array_like_reader.py b/aicsimageio/tests/readers/test_array_like_reader.py index f59f923f0..0736ad87d 100644 --- a/aicsimageio/tests/readers/test_array_like_reader.py +++ b/aicsimageio/tests/readers/test_array_like_reader.py @@ -8,7 +8,7 @@ import pytest import xarray as xr -from aicsimageio import exceptions, types +from aicsimageio import AICSImage, dimensions, exceptions, types from aicsimageio.readers import ArrayLikeReader from ..image_container_test_utils import run_image_container_checks @@ -857,3 +857,278 @@ def test_arraylike_reader( dict, ), ) + + +@pytest.mark.parametrize( + "image, " + "dim_order, " + "channel_names, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dims, " + "expected_channel_names", + [ + # Check no metadata provided 2D + # these are really simple just None checks + ( + np.random.rand(1, 1), + None, + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + ( + xr.DataArray(np.random.rand(1, 1)), + None, + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + # Check no metadata provided 4D + # these check that channel names are created for all + # and specifically for xr that channel names are overwritten + ( + np.random.rand(1, 1, 1, 1), + None, + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + ( + xr.DataArray(np.random.rand(1, 1, 1, 1)), + None, + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + # Test many scene, same dim_order, second scene + ( + [np.random.rand(1, 1, 1), np.random.rand(2, 2, 2)], + "CYX", + None, + "Image:1", + ("Image:0", "Image:1"), + (1, 2, 1, 2, 2), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:1:0", "Channel:1:1"], + ), + ( + [ + xr.DataArray(np.random.rand(1, 1, 1)), + xr.DataArray(np.random.rand(2, 2, 2)), + ], + "CYX", + None, + "Image:1", + ("Image:0", "Image:1"), + (1, 2, 1, 2, 2), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:1:0", "Channel:1:1"], + ), + # Test many scene, different dim_order, different channel_names, second scene + ( + [np.random.rand(1, 1, 1), np.random.rand(2, 2, 2)], + [None, "CYX"], + [None, ["A", "B"]], + "Image:1", + ("Image:0", "Image:1"), + (1, 2, 1, 2, 2), + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B"], + ), + ( + [ + xr.DataArray(np.random.rand(1, 1, 1)), + xr.DataArray(np.random.rand(2, 2, 2)), + ], + [None, "CYX"], + [None, ["A", "B"]], + "Image:1", + ("Image:0", "Image:1"), + (1, 2, 1, 2, 2), + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B"], + ), + # Test filled in xarray(s) + # no metadata should be overwritten + ( + xr.DataArray(np.random.rand(1, 1, 1), dims=list("TYX")), + None, + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + ( + xr.DataArray( + np.random.rand(1, 1, 1), dims=list("CYX"), coords={"C": ["A"]} + ), + None, + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["A"], + ), + ( + [ + xr.DataArray(np.random.rand(1, 1, 1), dims=list("TYX")), + xr.DataArray( + np.random.rand(2, 2, 2), dims=list("CYX"), coords={"C": ["A", "B"]} + ), + ], + None, + None, + "Image:0", + ("Image:0", "Image:1"), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + ( + [ + xr.DataArray(np.random.rand(1, 1, 1), dims=list("TYX")), + xr.DataArray( + np.random.rand(2, 2, 2), dims=list("CYX"), coords={"C": ["A", "B"]} + ), + ], + None, + None, + "Image:1", + ("Image:0", "Image:1"), + (1, 2, 1, 2, 2), + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B"], + ), + # Test non-standard dimensions + ( + np.random.rand(2, 2, 2), + "ABD", + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + ( + xr.DataArray(np.random.rand(2, 2, 2)), + "ABD", + None, + "Image:0", + ("Image:0",), + (1, 1, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0"], + ), + # Test that we can support many dimensions if dims is provided + ( + np.random.rand(1, 2, 3, 4, 5, 6, 7, 8), + "ABCDEFGH", + None, + "Image:0", + ("Image:0",), + (1, 3, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2"], + ), + ( + [ + np.random.rand(1, 2, 3, 4, 5, 6, 7, 8), + da.random.random((1, 2, 3, 4, 5, 6, 7, 8)), + ], + "ABCDEFGH", + None, + "Image:0", + ("Image:0", "Image:1"), + (1, 3, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2"], + ), + ( + [ + np.random.rand(1, 2, 3, 4, 5, 6, 7, 8), + da.random.random((2, 3, 4, 5, 6, 7, 8, 9)), + ], + "ABCDEFGH", + None, + "Image:1", + ("Image:0", "Image:1"), + (1, 4, 1, 1, 1), + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:1:0", "Channel:1:1", "Channel:1:2", "Channel:1:3"], + ), + # Test that without dims and with more than five dims, it raises an error + # Our guess dim order only support up to five dims + pytest.param( + np.random.rand(1, 2, 3, 4, 5, 6), + None, + None, + None, + None, + None, + None, + None, + marks=pytest.mark.raises( + exceptions=exceptions.InvalidDimensionOrderingError + ), + ), + pytest.param( + "hello world", + None, + None, + None, + None, + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.UnsupportedFileFormatError), + ), + ], +) +def test_aicsimage_from_array( + image: Union[types.MetaArrayLike, List[types.MetaArrayLike]], + dim_order: Optional[str], + channel_names: Optional[List[str]], + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dims: str, + expected_channel_names: List[str], +) -> None: + # Init + image_container = AICSImage(image, dim_order=dim_order, channel_names=channel_names) + + run_image_container_checks( + image_container=image_container, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=np.dtype(np.float64), + expected_dims_order=expected_dims, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=(None, None, None), + # we allow both None and Dict because the user can pass an already initialized + # xarray DataArray which has metadata as a dict + expected_metadata_type=( + type(None), + dict, + ), + ) diff --git a/aicsimageio/tests/readers/test_default_reader.py b/aicsimageio/tests/readers/test_default_reader.py deleted file mode 100644 index 9b628b625..000000000 --- a/aicsimageio/tests/readers/test_default_reader.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from typing import Tuple - -import numpy as np -import pytest - -from aicsimageio import exceptions -from aicsimageio.readers.default_reader import DefaultReader - -from ..conftest import get_resource_full_path, host -from ..image_container_test_utils import run_image_file_checks - - -@host -@pytest.mark.parametrize( - "filename, set_scene, expected_shape, expected_dims_order", - [ - ("example.bmp", "Image:0", (480, 640, 4), "YXS"), - ("example.png", "Image:0", (800, 537, 4), "YXS"), - ("example.jpg", "Image:0", (452, 400, 3), "YXS"), - ("example.gif", "Image:0", (72, 268, 268, 4), "TYXS"), - ( - "example_invalid_frame_count.mp4", - "Image:0", - (55, 1080, 1920, 3), - "TYXS", - ), - ( - "example_valid_frame_count.mp4", - "Image:0", - (72, 272, 272, 3), - "TYXS", - ), - pytest.param( - "example.txt", - None, - None, - None, - marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError), - ), - pytest.param( - "example.png", - "Image:1", - None, - None, - marks=pytest.mark.raises(exception=IndexError), - ), - ], -) -def test_default_reader( - filename: str, - host: str, - set_scene: str, - expected_shape: Tuple[int, ...], - expected_dims_order: str, -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, host) - - # Run checks - run_image_file_checks( - ImageContainer=DefaultReader, - image=uri, - set_scene=set_scene, - expected_scenes=("Image:0",), - expected_current_scene="Image:0", - expected_shape=expected_shape, - expected_dtype=np.dtype(np.uint8), - expected_dims_order=expected_dims_order, - expected_channel_names=None, - expected_physical_pixel_sizes=(None, None, None), - expected_metadata_type=dict, - ) - - -def test_ffmpeg_header_fail() -> None: - with pytest.raises(IOError): - # Big Buck Bunny - DefaultReader("https://archive.org/embed/archive-video-files/test.mp4") diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index b2f63fec4..9d9e9495c 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -51,7 +51,7 @@ def make_fake_data_2d(path: Path, as_mm: bool = False) -> xr.DataArray: name = f"img_channel{c}_position{s}_time{t}_z{z}.tif" else: name = f"S{s}_T{t}_C{c}_Z{z}.tif" - tiff.imsave( + tiff.imwrite( str(path / "2d_images" / name), im, dtype=np.uint16, @@ -90,7 +90,7 @@ def make_fake_data_3d(path: Path) -> xr.DataArray: shape_for_3d = (*DATA_SHAPE[:3], int(DATA_SHAPE[3] / 2)) for s, t, c, z in product(*(range(x) for x in shape_for_3d)): im = data[s, t, c, 2 * z : 2 * (z + 1)] - tiff.imsave( + tiff.imwrite( str(path / f"3d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16, @@ -149,7 +149,7 @@ def make_fake_data_4d(path: Path) -> xr.DataArray: c, per_file_z * z : per_file_z * (z + 1), ] - tiff.imsave( + tiff.imwrite( str(path / f"4d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16, diff --git a/aicsimageio/tests/readers/test_ome_tiff_reader.py b/aicsimageio/tests/readers/test_ome_tiff_reader.py index 959c12429..69532a8f6 100644 --- a/aicsimageio/tests/readers/test_ome_tiff_reader.py +++ b/aicsimageio/tests/readers/test_ome_tiff_reader.py @@ -1,14 +1,19 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List, Tuple +import time +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, List, Optional, Tuple, Type, Union import numpy as np import pytest +from distributed import Client, LocalCluster from ome_types import OME -from aicsimageio import dimensions, exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers import OmeTiffReader +from aicsimageio.readers.reader import Reader from ..conftest import LOCAL, get_resource_full_path, host from ..image_container_test_utils import ( @@ -470,3 +475,396 @@ def test_micromanager_ome_tiff_main_file() -> None: # expected_channel_names=["Cy5", "DAPI", "FITC"], # expected_physical_pixel_sizes=(1.75, 2.0, 2.0), # ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "s_1_t_1_c_10_z_1.ome.tiff", + "Image:0", + ("Image:0",), + (1, 10, 1, 1736, 1776), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + [f"C:{i}" for i in range(10)], # This is the actual metadata + (None, None, None), + OME, + ), + ( + # This is actually an OME-TIFF file + # Shows we don't just work off of extensions + # But the content of the file + "s_1_t_1_c_2_z_1_RGB.tiff", + "Image:0", + ("Image:0",), + (1, 2, 1, 32, 32, 3), + np.uint8, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0", "Channel:0:1"], + (None, None, None), + OME, + ), + ( + "s_3_t_1_c_3_z_5.ome.tiff", + "Image:0", + ("Image:0", "Image:1", "Image:2"), + (1, 3, 5, 325, 475), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["EGFP", "TaRFP", "Bright"], + (1.0, 1.0833333333333333, 1.0833333333333333), + OME, + ), + ( + "actk.ome.tiff", + "Image:0", + ("Image:0",), + (1, 6, 65, 233, 345), + np.float64, + dimensions.DEFAULT_DIMENSION_ORDER, + [ + "nucleus_segmentation", + "membrane_segmentation", + "dna", + "membrane", + "structure", + "brightfield", + ], + (0.29, 0.29, 0.29), + OME, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) + + +@pytest.mark.parametrize( + "filename, " + "first_scene_id, " + "first_scene_shape, " + "second_scene_id, " + "second_scene_shape", + [ + ( + "s_3_t_1_c_3_z_5.ome.tiff", + "Image:0", + (1, 3, 5, 325, 475), + "Image:1", + (1, 3, 5, 325, 475), + ), + ( + "s_3_t_1_c_3_z_5.ome.tiff", + "Image:1", + (1, 3, 5, 325, 475), + "Image:2", + (1, 3, 5, 325, 475), + ), + pytest.param( + "s_1_t_1_c_1_z_1.ome.tiff", + "Image:1", + None, + None, + None, + marks=pytest.mark.raises(exception=IndexError), + ), + pytest.param( + "s_3_t_1_c_3_z_5.ome.tiff", + "Image:3", + None, + None, + None, + marks=pytest.mark.raises(exception=IndexError), + ), + ], +) +def test_multi_scene_aicsimage( + filename: str, + first_scene_id: str, + first_scene_shape: Tuple[int, ...], + second_scene_id: str, + second_scene_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_multi_scene_image_read_checks( + ImageContainer=AICSImage, + image=uri, + first_scene_id=first_scene_id, + first_scene_shape=first_scene_shape, + first_scene_dtype=np.dtype(np.uint16), + second_scene_id=second_scene_id, + second_scene_shape=second_scene_shape, + second_scene_dtype=np.dtype(np.uint16), + ) + + +@pytest.mark.parametrize( + "filename, set_scene, get_dims, get_specific_dims, expected_shape", + [ + # Check normal volumetric data + ( + "3d-cell-viewer.ome.tiff", + "Image:0", + "CZYX", + {"C": [0, 1, 2, 3]}, + (4, 74, 1024, 1024), + ), + ], +) +@pytest.mark.parametrize("chunk_dims", ["YX", "ZYX"]) +@pytest.mark.parametrize("processes", [True, False]) +def test_parallel_read( + filename: str, + set_scene: str, + chunk_dims: str, + processes: bool, + get_dims: str, + get_specific_dims: Dict[str, Union[int, slice, range, Tuple[int, ...], List[int]]], + expected_shape: Tuple[int, ...], +) -> None: + """ + This test ensures that our produced dask array can be read in parallel. + """ + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init image + img = AICSImage(uri, chunk_dims=chunk_dims) + img.set_scene(set_scene) + + # Init cluster + cluster = LocalCluster(processes=processes) + client = Client(cluster) + + # Select data + out = img.get_image_dask_data(get_dims, **get_specific_dims).compute() + assert out.shape == expected_shape + + # Shutdown and then safety measure timeout + cluster.close() + client.close() + time.sleep(5) + + +@pytest.mark.parametrize( + "filename, " + "reconstruct_mosaic, " + "set_scene, " + "expected_shape, " + "expected_mosaic_tile_dims, " + "specific_tile_index", + [ + pytest.param( + "actk.ome.tiff", + True, + "Image:0", + (1, 6, 65, 233, 345), + None, + 0, + # AttributeError raises not because of error in rollback + # but because cannot access Y or X from + # None return from `mosaic_tile_dims` because + # image is not a mosaic tiled image + marks=pytest.mark.raises(exception=AttributeError), + ), + ], +) +def test_mosaic_passthrough( + filename: str, + reconstruct_mosaic: bool, + set_scene: str, + expected_shape: Tuple[int, ...], + expected_mosaic_tile_dims: Tuple[int, int], + specific_tile_index: int, +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, reconstruct_mosaic=reconstruct_mosaic) + img.set_scene(set_scene) + + # Assert basics + assert img.shape == expected_shape + assert img.mosaic_tile_dims.Y == expected_mosaic_tile_dims[0] # type: ignore + assert img.mosaic_tile_dims.X == expected_mosaic_tile_dims[1] # type: ignore + + # Ensure that regardless of stitched or not, we can get tile position + img.get_mosaic_tile_position(specific_tile_index) + + +@pytest.mark.parametrize( + "filename, expected_reader", + [ + ( + "s_1_t_1_c_1_z_1.ome.tiff", + OmeTiffReader, + ), + ( # Multiscene tiff + "variable_scene_shape_first_scene_pyramid.ome.tiff", + OmeTiffReader, + ), + ], +) +def test_selected_tiff_reader( + filename: str, + expected_reader: Type[Reader], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri) + + # Assert basics + assert isinstance(img.reader, expected_reader) + + +@pytest.mark.parametrize( + "filename, set_reader, extra_kwargs, expected_dims, expected_shape", + [ + # Test incompatible reader + pytest.param( + "s_1_t_1_c_1_z_1.tiff", + OmeTiffReader, + {}, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.UnsupportedFileFormatError), + ), + ], +) +def test_set_reader( + filename: str, + set_reader: Type[Reader], + extra_kwargs: Dict[str, Any], + expected_dims: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, reader=set_reader, **extra_kwargs) + + # Assert basics + assert img.dims.order == expected_dims + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename, select_scenes", + [ + ("s_3_t_1_c_3_z_5.ome.tiff", None), + ("s_3_t_1_c_3_z_5.ome.tiff", ["Image:2", "Image:1"]), + ], +) +def test_roundtrip_save_all_scenes( + filename: str, select_scenes: Optional[List[str]] +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Read initial + original = AICSImage(uri) + + # Save to temp and compare + with TemporaryDirectory() as tmpdir: + save_path = Path(tmpdir) / f"converted-{filename}.ome.tiff" + original.save(save_path, select_scenes=select_scenes) + + # Re-read + result = AICSImage(save_path) + + # Compare all scenes + # They may not have the same scene ids as some readers use scene names as the + # id, see LifReader for example + if select_scenes is None: + select_original_scenes = list(original.scenes) + else: + select_original_scenes = select_scenes + + assert len(select_original_scenes) == len(result.scenes) + for original_scene_id, result_scene_id in zip( + select_original_scenes, result.scenes + ): + # Compare + original.set_scene(original_scene_id) + result.set_scene(result_scene_id) + + np.testing.assert_array_equal(original.data, result.data) + assert original.dims.order == result.dims.order + assert original.channel_names == result.channel_names + + +@pytest.mark.parametrize( + "filename, expected_shape", + [ + ("s_1_t_1_c_10_z_1.ome.tiff", (1, 10, 1, 1736, 1776)), + ], +) +def test_no_scene_prop_access( + filename: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Construct image and check no scene call with property access + img = AICSImage(uri) + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename", + [ + "actk.ome.tiff", + ], +) +def test_ome_metadata(filename: str) -> None: + # Get full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init image + img = AICSImage(uri) + + # Test the transform + assert isinstance(img.ome_metadata, OME) diff --git a/aicsimageio/tests/readers/test_tiff_reader.py b/aicsimageio/tests/readers/test_tiff_reader.py index 3d14d6f1b..244fa0221 100644 --- a/aicsimageio/tests/readers/test_tiff_reader.py +++ b/aicsimageio/tests/readers/test_tiff_reader.py @@ -1,15 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List, Tuple +import time +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, List, Optional, Tuple, Type, Union import numpy as np import pytest +from distributed import Client, LocalCluster -from aicsimageio import exceptions +from aicsimageio import AICSImage, dimensions, exceptions from aicsimageio.readers import TiffReader +from aicsimageio.readers.reader import Reader -from ..conftest import LOCAL, get_resource_full_path, host +from ..conftest import LOCAL, REMOTE, get_resource_full_path, host from ..image_container_test_utils import ( run_image_file_checks, run_multi_scene_image_read_checks, @@ -276,3 +281,470 @@ def test_micromanager_ome_tiff_binary_file() -> None: expected_physical_pixel_sizes=(None, None, None), expected_metadata_type=str, ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "expected_scenes, " + "expected_shape, " + "expected_dtype, " + "expected_dims_order, " + "expected_channel_names, " + "expected_physical_pixel_sizes, " + "expected_metadata_type", + [ + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + ("Image:0",), + (10, 3, 1, 325, 475), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2"], + (None, None, None), + str, + ), + ( + "s_1_t_1_c_1_z_1_RGB.tiff", + "Image:0", + ("Image:0",), + (1, 1, 1, 7548, 7548, 3), + np.uint16, + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, + ["Channel:0:0"], + (None, None, None), + str, + ), + ], +) +def test_aicsimage( + filename: str, + set_scene: str, + expected_scenes: Tuple[str, ...], + expected_shape: Tuple[int, ...], + expected_dtype: np.dtype, + expected_dims_order: str, + expected_channel_names: List[str], + expected_physical_pixel_sizes: Tuple[float, float, float], + expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + run_image_file_checks( + ImageContainer=AICSImage, + image=uri, + set_scene=set_scene, + expected_scenes=expected_scenes, + expected_current_scene=set_scene, + expected_shape=expected_shape, + expected_dtype=expected_dtype, + expected_dims_order=expected_dims_order, + expected_channel_names=expected_channel_names, + expected_physical_pixel_sizes=expected_physical_pixel_sizes, + expected_metadata_type=expected_metadata_type, + ) + + +@pytest.mark.parametrize( + "filename, " + "set_scene, " + "set_dims, " + "set_channel_names, " + "expected_dims, " + "expected_channel_names, " + "expected_shape", + [ + # First check to show nothing changes + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + None, + None, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2"], + (10, 3, 1, 325, 475), + ), + # Check just dims to see default channel name creation + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + "ZCYX", + None, + dimensions.DEFAULT_DIMENSION_ORDER, + ["Channel:0:0", "Channel:0:1", "Channel:0:2"], + (1, 3, 10, 325, 475), + ), + # Check setting both as simple definitions + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + "ZCYX", + ["A", "B", "C"], + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B", "C"], + (1, 3, 10, 325, 475), + ), + # Check setting channels as a list of lists definitions + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + "ZCYX", + [["A", "B", "C"]], + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B", "C"], + (1, 3, 10, 325, 475), + ), + # Check setting dims as list of dims + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + ["ZCYX"], + ["A", "B", "C"], + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B", "C"], + (1, 3, 10, 325, 475), + ), + # Check setting dims as list of None (scene has unknown dims) + ( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + [None], + ["A", "B", "C"], + dimensions.DEFAULT_DIMENSION_ORDER, + ["A", "B", "C"], + (10, 3, 1, 325, 475), + ), + # Check providing too many dims + pytest.param( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + "ABCDEFG", + None, + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + # Check providing too many channels + pytest.param( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + None, + ["A", "B", "C", "D"], + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + # Check providing channels but no channel dim + pytest.param( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + "TZYX", + ["A", "B", "C"], + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + # Check number of scenes dims list matches n scenes + pytest.param( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + ["ABC", "DEF", "GHI"], + None, + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + # Check number of scenes channels list matches n scenes + pytest.param( + "s_1_t_10_c_3_z_1.tiff", + "Image:0", + None, + [["A", "B", "C"], ["D", "E", "F"]], + None, + None, + None, + marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), + ), + ], +) +def test_set_coords( + filename: str, + set_scene: str, + set_dims: Optional[Union[str, List[str]]], + set_channel_names: Optional[Union[List[str], List[List[str]]]], + expected_dims: str, + expected_channel_names: List[str], + expected_shape: Tuple[int, ...], +) -> None: + # As a reminder, AICSImage always has certain dimensions + # If you provide a dimension that isn't one of those, + # it will only be available on the reader, not the AICSImage object. + + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, dim_order=set_dims, channel_names=set_channel_names) + + # Set scene + img.set_scene(set_scene) + + # Compare AICSImage results + assert img.dims.order == expected_dims + assert img.channel_names == expected_channel_names + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename, expected_reader", + [ + ( + "s_1_t_1_c_1_z_1.tiff", + TiffReader, + ), + ], +) +def test_selected_tiff_reader( + filename: str, + expected_reader: Type[Reader], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri) + + # Assert basics + assert isinstance(img.reader, expected_reader) + + +@pytest.mark.parametrize( + "filename, set_reader, extra_kwargs, expected_dims, expected_shape", + [ + ( + "actk.ome.tiff", + TiffReader, + {}, + dimensions.DEFAULT_DIMENSION_ORDER, + (1, 6, 65, 233, 345), + ), + # Test good reader but also allows extra kwargs + ( + "actk.ome.tiff", + TiffReader, + {"dim_order": "CTYX"}, + dimensions.DEFAULT_DIMENSION_ORDER, + (65, 6, 1, 233, 345), + ), + ], +) +def test_set_reader( + filename: str, + set_reader: Type[Reader], + extra_kwargs: Dict[str, Any], + expected_dims: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init + img = AICSImage(uri, reader=set_reader, **extra_kwargs) + + # Assert basics + assert img.dims.order == expected_dims + assert img.shape == expected_shape + + +@pytest.mark.parametrize( + "filename, set_scene, get_dims, get_specific_dims, expected_shape", + [ + ( + "s_1_t_1_c_2_z_1_RGB.tiff", + "Image:0", + "CYXS", + {}, + (2, 32, 32, 3), + ), + ], +) +@pytest.mark.parametrize("chunk_dims", ["YX", "ZYX"]) +@pytest.mark.parametrize("processes", [True, False]) +def test_parallel_read( + filename: str, + set_scene: str, + chunk_dims: str, + processes: bool, + get_dims: str, + get_specific_dims: Dict[str, Union[int, slice, range, Tuple[int, ...], List[int]]], + expected_shape: Tuple[int, ...], +) -> None: + """ + This test ensures that our produced dask array can be read in parallel. + """ + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Init image + img = AICSImage(uri, chunk_dims=chunk_dims) + img.set_scene(set_scene) + + # Init cluster + cluster = LocalCluster(processes=processes) + client = Client(cluster) + + # Select data + out = img.get_image_dask_data(get_dims, **get_specific_dims).compute() + assert out.shape == expected_shape + + # Shutdown and then safety measure timeout + cluster.close() + client.close() + time.sleep(5) + + +@pytest.mark.parametrize( + "filename, " + "host, " + "first_scene, " + "expected_first_chunk_shape, " + "second_scene, " + "expected_second_chunk_shape", + [ + ( + "image_stack_tpzc_50tp_2p_5z_3c_512k_1_MMStack_2-Pos000_000.ome.tif", + LOCAL, + 0, + (50, 5, 256, 256), + 1, + (50, 5, 256, 256), + ), + ( + "image_stack_tpzc_50tp_2p_5z_3c_512k_1_MMStack_2-Pos000_000.ome.tif", + LOCAL, + 1, + (50, 5, 256, 256), + 0, + (50, 5, 256, 256), + ), + pytest.param( + "image_stack_tpzc_50tp_2p_5z_3c_512k_1_MMStack_2-Pos000_000.ome.tif", + REMOTE, + 1, # Start with second scene to trigger error faster + (50, 5, 256, 256), + 0, + (50, 5, 256, 256), + marks=pytest.mark.raises(exceptions=IndexError), + ), + ], +) +@pytest.mark.parametrize("processes", [True, False]) +def test_parallel_multifile_tiff_read( + filename: str, + host: str, + first_scene: int, + expected_first_chunk_shape: Tuple[int, ...], + second_scene: int, + expected_second_chunk_shape: Tuple[int, ...], + processes: bool, +) -> None: + """ + This test ensures that we can serialize and read 'multi-file multi-scene' formats. + See: https://github.com/AllenCellModeling/aicsimageio/issues/196 + + We specifically test with a Distributed cluster to ensure that we serialize and + read properly from each file. + """ + # Construct full filepath + uri = get_resource_full_path(filename, host) + + # Init image + img = AICSImage(uri) + + # Init cluster + cluster = LocalCluster(processes=processes) + client = Client(cluster) + + # Select data + img.set_scene(first_scene) + first_out = img.get_image_dask_data("TZYX").compute() + assert first_out.shape == expected_first_chunk_shape + + # Update scene and select data + img.set_scene(second_scene) + second_out = img.get_image_dask_data("TZYX").compute() + assert second_out.shape == expected_second_chunk_shape + + # Shutdown and then safety measure timeout + cluster.close() + client.close() + time.sleep(5) + + +@pytest.mark.parametrize( + "filename, select_scenes", + [ + ("s_1_t_10_c_3_z_1.tiff", None), + ], +) +def test_roundtrip_save_all_scenes( + filename: str, select_scenes: Optional[List[str]] +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Read initial + original = AICSImage(uri) + + # Save to temp and compare + with TemporaryDirectory() as tmpdir: + save_path = Path(tmpdir) / f"converted-{filename}.ome.tiff" + original.save(save_path, select_scenes=select_scenes) + + # Re-read + result = AICSImage(save_path) + + # Compare all scenes + # They may not have the same scene ids as some readers use scene names as the + # id, see LifReader for example + if select_scenes is None: + select_original_scenes = list(original.scenes) + else: + select_original_scenes = select_scenes + + assert len(select_original_scenes) == len(result.scenes) + for original_scene_id, result_scene_id in zip( + select_original_scenes, result.scenes + ): + # Compare + original.set_scene(original_scene_id) + result.set_scene(result_scene_id) + + np.testing.assert_array_equal(original.data, result.data) + assert original.dims.order == result.dims.order + assert original.channel_names == result.channel_names + + +@pytest.mark.parametrize( + "filename, expected_shape", + [ + ("s_1_t_10_c_3_z_1.tiff", (10, 3, 1, 325, 475)), + ], +) +def test_no_scene_prop_access( + filename: str, + expected_shape: Tuple[int, ...], +) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Construct image and check no scene call with property access + img = AICSImage(uri) + assert img.shape == expected_shape diff --git a/aicsimageio/tests/test_aics_image.py b/aicsimageio/tests/test_aics_image.py index 5b5bbe1c4..329157336 100644 --- a/aicsimageio/tests/test_aics_image.py +++ b/aicsimageio/tests/test_aics_image.py @@ -1,1317 +1,29 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import time -import xml.etree.ElementTree as ET -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Any, Dict, List, Optional, Tuple, Type, Union - -import dask.array as da -import numpy as np import pytest -import xarray as xr -from distributed import Client, LocalCluster -from ome_types import OME - -from aicsimageio import AICSImage, dimensions, exceptions, readers, types - -from .conftest import LOCAL, REMOTE, get_resource_full_path -from .image_container_test_utils import ( - run_image_container_checks, - run_image_file_checks, - run_multi_scene_image_read_checks, -) -############################################################################### +from aicsimageio import AICSImage, exceptions -# We only run checks on a subset of local files -# The base reader unit tests show that we can handle both local or remote -# If we handled them here, tests would just take longer than they already do - -############################################################################### +from .conftest import LOCAL, get_resource_full_path @pytest.mark.parametrize( - "filename, " - "set_scene, " - "expected_scenes, " - "expected_shape, " - "expected_dtype, " - "expected_dims_order, " - "expected_channel_names, " - "expected_physical_pixel_sizes, " - "expected_metadata_type", + "filename", [ - ####################################################################### - # DefaultReader - ( - "example.png", - "Image:0", - ("Image:0",), - (1, 1, 1, 800, 537, 4), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0"], - (None, None, None), - dict, - ), - ( - "example.gif", - "Image:0", - ("Image:0",), - (72, 1, 1, 268, 268, 4), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0"], - (None, None, None), - dict, - ), - ( - "example_valid_frame_count.mp4", - "Image:0", - ("Image:0",), - (72, 1, 1, 272, 272, 3), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0"], - (None, None, None), - dict, - ), - ####################################################################### - # TiffReader - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - ("Image:0",), - (10, 3, 1, 325, 475), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2"], - (None, None, None), - str, - ), - ( - "s_1_t_1_c_1_z_1_RGB.tiff", - "Image:0", - ("Image:0",), - (1, 1, 1, 7548, 7548, 3), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0"], - (None, None, None), - str, - ), - ####################################################################### - # OmeTiffReader - ( - "s_1_t_1_c_10_z_1.ome.tiff", - "Image:0", - ("Image:0",), - (1, 10, 1, 1736, 1776), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - [f"C:{i}" for i in range(10)], # This is the actual metadata - (None, None, None), - OME, - ), - ( - # This is actually an OME-TIFF file - # Shows we don't just work off of extensions - # But the content of the file - "s_1_t_1_c_2_z_1_RGB.tiff", - "Image:0", - ("Image:0",), - (1, 2, 1, 32, 32, 3), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0", "Channel:0:1"], - (None, None, None), - OME, - ), - ( - "s_3_t_1_c_3_z_5.ome.tiff", - "Image:0", - ("Image:0", "Image:1", "Image:2"), - (1, 3, 5, 325, 475), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["EGFP", "TaRFP", "Bright"], - (1.0, 1.0833333333333333, 1.0833333333333333), - OME, - ), - ( - "actk.ome.tiff", - "Image:0", - ("Image:0",), - (1, 6, 65, 233, 345), - np.float64, - dimensions.DEFAULT_DIMENSION_ORDER, - [ - "nucleus_segmentation", - "membrane_segmentation", - "dna", - "membrane", - "structure", - "brightfield", - ], - (0.29, 0.29, 0.29), - OME, - ), - ####################################################################### - # LifReader - ( - "s_1_t_4_c_2_z_1.lif", - "b2_001_Crop001_Resize001", - ("b2_001_Crop001_Resize001",), - (4, 2, 1, 614, 614), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Gray--TL-PH--EMP_BF", "Green--FLUO--GFP"], - (None, 0.33914910277324634, 0.33914910277324634), - ET.Element, - ), - ( - "tiled.lif", - "TileScan_002", - ("TileScan_002",), - (1, 4, 1, 5622, 7666), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Gray", "Red", "Green", "Cyan"], - (None, 0.20061311154598827, 0.20061311154598827), - ET.Element, - ), - ####################################################################### - # CziReader - ( - "s_1_t_1_c_1_z_1.czi", - "Image:0", - ("Image:0",), - (1, 1, 1, 325, 475), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Bright"], - (None, 1.0833333333333333, 1.0833333333333333), - ET.Element, - ), - ( - "RGB-8bit.czi", - "Image:0", - ("Image:0",), - (1, 1, 1, 624, 924, 3), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0"], - (None, 1.0833333333333333, 1.0833333333333333), - ET.Element, - ), - ####################################################################### - # DV-Reader - ( - "DV_siRNAi-HeLa_IN_02.r3d_D3D.dv", - "Image:0", - ("Image:0",), - (1, 4, 40, 512, 512), - np.int16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["360/457", "490/528", "555/617", "640/685"], - (0.20000000298023224, 0.06502940505743027, 0.06502940505743027), - dict, - ), - ####################################################################### - # ND2 Reader - ( - "ND2_jonas_header_test2.nd2", - "XYPos:0", - ("XYPos:0",), - (4, 1, 5, 520, 696), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Jonas_DIC"], - (0.5, 0.12863494437945, 0.12863494437945), - dict, - ), - ####################################################################### - # BioformatsReader - ( - "Olympus-OIR_etienne_amy_slice_z_stack_0001.oir", - "Olympus-OIR_etienne_amy_slice_z_stack_0001.oir", - ("Olympus-OIR_etienne_amy_slice_z_stack_0001.oir",), - (32, 1, 1, 512, 512), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["CH3"], - (1.0, 1.242961138804478, 1.242961138804478), - OME, - ), - ( - "Imaris-IMS_davemason_Convallaria_3C_1T_confocal.ims", - "Imaris-IMS_davemason_Convallaria_3C_1T_confocal.ims Resolution Level 1", - ("Imaris-IMS_davemason_Convallaria_3C_1T_confocal.ims Resolution Level 1",), - (1, 3, 1, 1024, 1024), - np.uint16, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2"], - (0.001, 1.2059374999999999, 1.2059570312500014), - OME, - ), - ( - "DICOM_samples_MR-MONO2-8-16x-heart.dcm", - "Series 0", - ("Series 0",), - (1, 1, 16, 256, 256), - np.uint8, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - (None, None, None), - OME, - ), - ####################################################################### - # Errors pytest.param( "example.txt", - None, - None, - None, - None, - None, - None, - None, - None, marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError), ), - pytest.param( - "s_1_t_1_c_1_z_1.ome.tiff", - "Image:1", - None, - None, - None, - None, - None, - None, - None, - marks=pytest.mark.raises(exception=IndexError), - ), - pytest.param( - "s_3_t_1_c_3_z_5.ome.tiff", - "Image:3", - None, - None, - None, - None, - None, - None, - None, - marks=pytest.mark.raises(exception=IndexError), - ), pytest.param( "does-not-exist-klafjjksdafkjl.bad", - None, - None, - None, - None, - None, - None, - None, - None, marks=pytest.mark.raises(exception=FileNotFoundError), ), ], ) def test_aicsimage( filename: str, - set_scene: str, - expected_scenes: Tuple[str, ...], - expected_shape: Tuple[int, ...], - expected_dtype: np.dtype, - expected_dims_order: str, - expected_channel_names: List[str], - expected_physical_pixel_sizes: Tuple[float, float, float], - expected_metadata_type: Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]], -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Run checks - run_image_file_checks( - ImageContainer=AICSImage, - image=uri, - set_scene=set_scene, - expected_scenes=expected_scenes, - expected_current_scene=set_scene, - expected_shape=expected_shape, - expected_dtype=expected_dtype, - expected_dims_order=expected_dims_order, - expected_channel_names=expected_channel_names, - expected_physical_pixel_sizes=expected_physical_pixel_sizes, - expected_metadata_type=expected_metadata_type, - ) - - -@pytest.mark.parametrize( - "filename, expected_shape", - [ - ("example.png", (1, 1, 1, 800, 537, 4)), - ("s_1_t_10_c_3_z_1.tiff", (10, 3, 1, 325, 475)), - ("s_1_t_1_c_10_z_1.ome.tiff", (1, 10, 1, 1736, 1776)), - ("s_1_t_4_c_2_z_1.lif", (4, 2, 1, 614, 614)), - ("RGB-8bit.czi", (1, 1, 1, 624, 924, 3)), - ], -) -def test_no_scene_prop_access( - filename: str, - expected_shape: Tuple[int, ...], -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Construct image and check no scene call with property access - img = AICSImage(uri) - assert img.shape == expected_shape - - -@pytest.mark.parametrize( - "filename, " - "first_scene_id, " - "first_scene_shape, " - "second_scene_id, " - "second_scene_shape", - [ - ( - "s_3_t_1_c_3_z_5.ome.tiff", - "Image:0", - (1, 3, 5, 325, 475), - "Image:1", - (1, 3, 5, 325, 475), - ), - ( - "s_3_t_1_c_3_z_5.ome.tiff", - "Image:1", - (1, 3, 5, 325, 475), - "Image:2", - (1, 3, 5, 325, 475), - ), - ( - "s_3_t_1_c_3_z_5.czi", - "P2", - (1, 3, 5, 325, 475), - "P3", - (1, 3, 5, 325, 475), - ), - ( - "s_3_t_1_c_3_z_5.czi", - "P3", - (1, 3, 5, 325, 475), - "P1", - (1, 3, 5, 325, 475), - ), - ( - "s_3_t_1_c_3_z_5.czi", - 1, - (1, 3, 5, 325, 475), - 2, - (1, 3, 5, 325, 475), - ), - pytest.param( - "s_3_t_1_c_3_z_5.czi", - ["this is not a scene id"], - None, - None, - None, - marks=pytest.mark.raises(exception=TypeError), - ), - ], -) -def test_multi_scene_aicsimage( - filename: str, - first_scene_id: str, - first_scene_shape: Tuple[int, ...], - second_scene_id: str, - second_scene_shape: Tuple[int, ...], -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Run checks - run_multi_scene_image_read_checks( - ImageContainer=AICSImage, - image=uri, - first_scene_id=first_scene_id, - first_scene_shape=first_scene_shape, - first_scene_dtype=np.dtype(np.uint16), - second_scene_id=second_scene_id, - second_scene_shape=second_scene_shape, - second_scene_dtype=np.dtype(np.uint16), - ) - - -@pytest.mark.parametrize( - "image, " - "dim_order, " - "channel_names, " - "set_scene, " - "expected_scenes, " - "expected_shape, " - "expected_dims, " - "expected_channel_names", - [ - # Check no metadata provided 2D - # these are really simple just None checks - ( - np.random.rand(1, 1), - None, - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - ( - xr.DataArray(np.random.rand(1, 1)), - None, - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - # Check no metadata provided 4D - # these check that channel names are created for all - # and specifically for xr that channel names are overwritten - ( - np.random.rand(1, 1, 1, 1), - None, - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - ( - xr.DataArray(np.random.rand(1, 1, 1, 1)), - None, - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - # Test many scene, same dim_order, second scene - ( - [np.random.rand(1, 1, 1), np.random.rand(2, 2, 2)], - "CYX", - None, - "Image:1", - ("Image:0", "Image:1"), - (1, 2, 1, 2, 2), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:1:0", "Channel:1:1"], - ), - ( - [ - xr.DataArray(np.random.rand(1, 1, 1)), - xr.DataArray(np.random.rand(2, 2, 2)), - ], - "CYX", - None, - "Image:1", - ("Image:0", "Image:1"), - (1, 2, 1, 2, 2), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:1:0", "Channel:1:1"], - ), - # Test many scene, different dim_order, different channel_names, second scene - ( - [np.random.rand(1, 1, 1), np.random.rand(2, 2, 2)], - [None, "CYX"], - [None, ["A", "B"]], - "Image:1", - ("Image:0", "Image:1"), - (1, 2, 1, 2, 2), - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B"], - ), - ( - [ - xr.DataArray(np.random.rand(1, 1, 1)), - xr.DataArray(np.random.rand(2, 2, 2)), - ], - [None, "CYX"], - [None, ["A", "B"]], - "Image:1", - ("Image:0", "Image:1"), - (1, 2, 1, 2, 2), - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B"], - ), - # Test filled in xarray(s) - # no metadata should be overwritten - ( - xr.DataArray(np.random.rand(1, 1, 1), dims=list("TYX")), - None, - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - ( - xr.DataArray( - np.random.rand(1, 1, 1), dims=list("CYX"), coords={"C": ["A"]} - ), - None, - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["A"], - ), - ( - [ - xr.DataArray(np.random.rand(1, 1, 1), dims=list("TYX")), - xr.DataArray( - np.random.rand(2, 2, 2), dims=list("CYX"), coords={"C": ["A", "B"]} - ), - ], - None, - None, - "Image:0", - ("Image:0", "Image:1"), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - ( - [ - xr.DataArray(np.random.rand(1, 1, 1), dims=list("TYX")), - xr.DataArray( - np.random.rand(2, 2, 2), dims=list("CYX"), coords={"C": ["A", "B"]} - ), - ], - None, - None, - "Image:1", - ("Image:0", "Image:1"), - (1, 2, 1, 2, 2), - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B"], - ), - # Test non-standard dimensions - ( - np.random.rand(2, 2, 2), - "ABD", - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - ( - xr.DataArray(np.random.rand(2, 2, 2)), - "ABD", - None, - "Image:0", - ("Image:0",), - (1, 1, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0"], - ), - # Test that we can support many dimensions if dims is provided - ( - np.random.rand(1, 2, 3, 4, 5, 6, 7, 8), - "ABCDEFGH", - None, - "Image:0", - ("Image:0",), - (1, 3, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2"], - ), - ( - [ - np.random.rand(1, 2, 3, 4, 5, 6, 7, 8), - da.random.random((1, 2, 3, 4, 5, 6, 7, 8)), - ], - "ABCDEFGH", - None, - "Image:0", - ("Image:0", "Image:1"), - (1, 3, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2"], - ), - ( - [ - np.random.rand(1, 2, 3, 4, 5, 6, 7, 8), - da.random.random((2, 3, 4, 5, 6, 7, 8, 9)), - ], - "ABCDEFGH", - None, - "Image:1", - ("Image:0", "Image:1"), - (1, 4, 1, 1, 1), - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:1:0", "Channel:1:1", "Channel:1:2", "Channel:1:3"], - ), - # Test that without dims and with more than five dims, it raises an error - # Our guess dim order only support up to five dims - pytest.param( - np.random.rand(1, 2, 3, 4, 5, 6), - None, - None, - None, - None, - None, - None, - None, - marks=pytest.mark.raises( - exceptions=exceptions.InvalidDimensionOrderingError - ), - ), - pytest.param( - "hello world", - None, - None, - None, - None, - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.UnsupportedFileFormatError), - ), - ], -) -def test_aicsimage_from_array( - image: Union[types.MetaArrayLike, List[types.MetaArrayLike]], - dim_order: Optional[str], - channel_names: Optional[List[str]], - set_scene: str, - expected_scenes: Tuple[str, ...], - expected_shape: Tuple[int, ...], - expected_dims: str, - expected_channel_names: List[str], -) -> None: - # Init - image_container = AICSImage(image, dim_order=dim_order, channel_names=channel_names) - - run_image_container_checks( - image_container=image_container, - set_scene=set_scene, - expected_scenes=expected_scenes, - expected_current_scene=set_scene, - expected_shape=expected_shape, - expected_dtype=np.dtype(np.float64), - expected_dims_order=expected_dims, - expected_channel_names=expected_channel_names, - expected_physical_pixel_sizes=(None, None, None), - # we allow both None and Dict because the user can pass an already initialized - # xarray DataArray which has metadata as a dict - expected_metadata_type=( - type(None), - dict, - ), - ) - - -@pytest.mark.parametrize( - "filename, select_scenes", - [ - ("s_1_t_10_c_3_z_1.tiff", None), - ("s_3_t_1_c_3_z_5.ome.tiff", None), - ("s_3_t_1_c_3_z_5.ome.tiff", ["Image:2", "Image:1"]), - ("s_1_t_4_c_2_z_1.lif", None), - ("tiled.lif", None), - ("s_1_t_1_c_1_z_1.czi", None), - ("s_3_t_1_c_3_z_5.czi", ["P2", "P1"]), - ], -) -def test_roundtrip_save_all_scenes( - filename: str, select_scenes: Optional[List[str]] -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Read initial - original = AICSImage(uri) - - # Save to temp and compare - with TemporaryDirectory() as tmpdir: - save_path = Path(tmpdir) / f"converted-{filename}.ome.tiff" - original.save(save_path, select_scenes=select_scenes) - - # Re-read - result = AICSImage(save_path) - - # Compare all scenes - # They may not have the same scene ids as some readers use scene names as the - # id, see LifReader for example - if select_scenes is None: - select_original_scenes = list(original.scenes) - else: - select_original_scenes = select_scenes - - assert len(select_original_scenes) == len(result.scenes) - for original_scene_id, result_scene_id in zip( - select_original_scenes, result.scenes - ): - # Compare - original.set_scene(original_scene_id) - result.set_scene(result_scene_id) - - np.testing.assert_array_equal(original.data, result.data) - assert original.dims.order == result.dims.order - assert original.channel_names == result.channel_names - - -@pytest.mark.parametrize( - "filename, " - "set_scene, " - "set_dims, " - "set_channel_names, " - "expected_dims, " - "expected_channel_names, " - "expected_shape", - [ - # DefaultReader - # First check to show nothing changes - ( - "example.gif", - "Image:0", - None, - None, - dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES, - ["Channel:0:0"], - (72, 1, 1, 268, 268, 4), - ), - # Check just dims to see default channel name creation - ( - "example.gif", - "Image:0", - "ZYXC", - None, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2", "Channel:0:3"], - (1, 4, 72, 268, 268), - ), - # Check setting both as simple definitions - ( - "example.gif", - "Image:0", - "ZYXC", - ["Red", "Green", "Blue", "Alpha"], - dimensions.DEFAULT_DIMENSION_ORDER, - ["Red", "Green", "Blue", "Alpha"], - (1, 4, 72, 268, 268), - ), - # Check providing too many dims - pytest.param( - "example.gif", - "Image:0", - "ABCDEFG", - None, - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - # Check providing too many channels - pytest.param( - "example.gif", - "Image:0", - "ZYXC", - ["A", "B", "C"], - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - # Check providing channels but no channel dim - pytest.param( - "example.gif", - "Image:0", - None, - ["A", "B", "C"], - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - ###################################### - # TiffReader - # First check to show nothing changes - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - None, - None, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2"], - (10, 3, 1, 325, 475), - ), - # Check just dims to see default channel name creation - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - "ZCYX", - None, - dimensions.DEFAULT_DIMENSION_ORDER, - ["Channel:0:0", "Channel:0:1", "Channel:0:2"], - (1, 3, 10, 325, 475), - ), - # Check setting both as simple definitions - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - "ZCYX", - ["A", "B", "C"], - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B", "C"], - (1, 3, 10, 325, 475), - ), - # Check setting channels as a list of lists definitions - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - "ZCYX", - [["A", "B", "C"]], - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B", "C"], - (1, 3, 10, 325, 475), - ), - # Check setting dims as list of dims - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - ["ZCYX"], - ["A", "B", "C"], - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B", "C"], - (1, 3, 10, 325, 475), - ), - # Check setting dims as list of None (scene has unknown dims) - ( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - [None], - ["A", "B", "C"], - dimensions.DEFAULT_DIMENSION_ORDER, - ["A", "B", "C"], - (10, 3, 1, 325, 475), - ), - # Check providing too many dims - pytest.param( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - "ABCDEFG", - None, - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - # Check providing too many channels - pytest.param( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - None, - ["A", "B", "C", "D"], - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - # Check providing channels but no channel dim - pytest.param( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - "TZYX", - ["A", "B", "C"], - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - # Check number of scenes dims list matches n scenes - pytest.param( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - ["ABC", "DEF", "GHI"], - None, - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - # Check number of scenes channels list matches n scenes - pytest.param( - "s_1_t_10_c_3_z_1.tiff", - "Image:0", - None, - [["A", "B", "C"], ["D", "E", "F"]], - None, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.ConflictingArgumentsError), - ), - ], -) -def test_set_coords( - filename: str, - set_scene: str, - set_dims: Optional[Union[str, List[str]]], - set_channel_names: Optional[Union[List[str], List[List[str]]]], - expected_dims: str, - expected_channel_names: List[str], - expected_shape: Tuple[int, ...], -) -> None: - # As a reminder, AICSImage always has certain dimensions - # If you provide a dimension that isn't one of those, - # it will only be available on the reader, not the AICSImage object. - - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Init - img = AICSImage(uri, dim_order=set_dims, channel_names=set_channel_names) - - # Set scene - img.set_scene(set_scene) - - # Compare AICSImage results - assert img.dims.order == expected_dims - assert img.channel_names == expected_channel_names - assert img.shape == expected_shape - - -@pytest.mark.parametrize( - "filename, expected_reader", - [ - ( - "s_1_t_1_c_1_z_1.tiff", - readers.TiffReader, - ), - ( - "s_1_t_1_c_1_z_1.ome.tiff", - readers.OmeTiffReader, - ), - ( - "s_1_t_1_c_1_z_1_ome_tiff_tiles.ome.tif", - readers.OmeTiledTiffReader, - ), - ( # Multiscene tiff - "variable_scene_shape_first_scene_pyramid.ome.tiff", - readers.OmeTiffReader, - ), - ], -) -def test_selected_tiff_reader( - filename: str, - expected_reader: Type[readers.reader.Reader], -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Init - img = AICSImage(uri) - - # Assert basics - assert isinstance(img.reader, expected_reader) - - -@pytest.mark.parametrize( - "filename, set_reader, extra_kwargs, expected_dims, expected_shape", - [ - ( - "actk.ome.tiff", - readers.TiffReader, - {}, - dimensions.DEFAULT_DIMENSION_ORDER, - (1, 6, 65, 233, 345), - ), - # See shape to see why you should use TiffReader :) - ( - "actk.ome.tiff", - readers.default_reader.DefaultReader, - {}, - dimensions.DEFAULT_DIMENSION_ORDER, - (390, 1, 1, 233, 345), - ), - # Test good reader but also allows extra kwargs - ( - "actk.ome.tiff", - readers.TiffReader, - {"dim_order": "CTYX"}, - dimensions.DEFAULT_DIMENSION_ORDER, - (65, 6, 1, 233, 345), - ), - # Test incompatible reader - pytest.param( - "actk.ome.tiff", - readers.lif_reader.LifReader, - {}, - None, - None, - marks=pytest.mark.raises(exceptions=exceptions.UnsupportedFileFormatError), - ), - ], -) -def test_set_reader( - filename: str, - set_reader: Type[readers.reader.Reader], - extra_kwargs: Dict[str, Any], - expected_dims: str, - expected_shape: Tuple[int, ...], -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Init - img = AICSImage(uri, reader=set_reader, **extra_kwargs) - - # Assert basics - assert img.dims.order == expected_dims - assert img.shape == expected_shape - - -@pytest.mark.parametrize( - "filename, " - "reconstruct_mosaic, " - "set_scene, " - "expected_shape, " - "expected_mosaic_tile_dims, " - "specific_tile_index", - [ - ( - "tiled.lif", - True, - "TileScan_002", - (1, 4, 1, 5622, 7666), - (512, 512), - 0, - ), - ( - "tiled.lif", - False, - "TileScan_002", - (165, 1, 4, 1, 512, 512), - (512, 512), - 0, - ), - ( - "OverViewScan.czi", - True, - "TR1", - (1, 1, 1, 3212, 7398), - (440, 544), - 0, - ), - ( - "OverViewScan.czi", - False, - "TR1", - (120, 1, 1, 1, 440, 544), - (440, 544), - 0, - ), - pytest.param( - "tiled.lif", - False, - "TileScan_002", - (165, 1, 4, 1, 512, 512), - (512, 512), - 999, - marks=pytest.mark.raises(exception=IndexError), - ), - pytest.param( - "actk.ome.tiff", - True, - "Image:0", - (1, 6, 65, 233, 345), - None, - 0, - # AttributeError raises not because of error in rollback - # but because cannot access Y or X from - # None return from `mosaic_tile_dims` because - # image is not a mosaic tiled image - marks=pytest.mark.raises(exception=AttributeError), - ), - ], -) -def test_mosaic_passthrough( - filename: str, - reconstruct_mosaic: bool, - set_scene: str, - expected_shape: Tuple[int, ...], - expected_mosaic_tile_dims: Tuple[int, int], - specific_tile_index: int, -) -> None: - # Construct full filepath - uri = get_resource_full_path(filename, LOCAL) - - # Init - img = AICSImage(uri, reconstruct_mosaic=reconstruct_mosaic) - img.set_scene(set_scene) - - # Assert basics - assert img.shape == expected_shape - assert img.mosaic_tile_dims.Y == expected_mosaic_tile_dims[0] # type: ignore - assert img.mosaic_tile_dims.X == expected_mosaic_tile_dims[1] # type: ignore - - # Ensure that regardless of stitched or not, we can get tile position - img.get_mosaic_tile_position(specific_tile_index) - - -@pytest.mark.parametrize( - "filename, set_scene, get_dims, get_specific_dims, expected_shape", - [ - # Check normal volumetric data - ( - "3d-cell-viewer.ome.tiff", - "Image:0", - "CZYX", - {"C": [0, 1, 2, 3]}, - (4, 74, 1024, 1024), - ), - ( - "s_1_t_1_c_2_z_1_RGB.tiff", - "Image:0", - "CYXS", - {}, - (2, 32, 32, 3), - ), - ( - "s_1_t_4_c_2_z_1.lif", - "b2_001_Crop001_Resize001", - "TYX", - {}, - (4, 614, 614), - ), - # Check mosaic chunk handling - ( - "tiled.lif", - "TileScan_002", - "CYX", - {"Y": slice(0, 2000), "X": slice(0, 2000)}, - (4, 2000, 2000), - ), - ], -) -@pytest.mark.parametrize("chunk_dims", ["YX", "ZYX"]) -@pytest.mark.parametrize("processes", [True, False]) -def test_parallel_read( - filename: str, - set_scene: str, - chunk_dims: str, - processes: bool, - get_dims: str, - get_specific_dims: Dict[str, Union[int, slice, range, Tuple[int, ...], List[int]]], - expected_shape: Tuple[int, ...], ) -> None: - """ - This test ensures that our produced dask array can be read in parallel. - """ # Construct full filepath uri = get_resource_full_path(filename, LOCAL) - - # Init image - img = AICSImage(uri, chunk_dims=chunk_dims) - img.set_scene(set_scene) - - # Init cluster - cluster = LocalCluster(processes=processes) - client = Client(cluster) - - # Select data - out = img.get_image_dask_data(get_dims, **get_specific_dims).compute() - assert out.shape == expected_shape - - # Shutdown and then safety measure timeout - cluster.close() - client.close() - time.sleep(5) - - -@pytest.mark.parametrize( - "filename, " - "host, " - "first_scene, " - "expected_first_chunk_shape, " - "second_scene, " - "expected_second_chunk_shape", - [ - ( - "image_stack_tpzc_50tp_2p_5z_3c_512k_1_MMStack_2-Pos000_000.ome.tif", - LOCAL, - 0, - (50, 5, 256, 256), - 1, - (50, 5, 256, 256), - ), - ( - "image_stack_tpzc_50tp_2p_5z_3c_512k_1_MMStack_2-Pos000_000.ome.tif", - LOCAL, - 1, - (50, 5, 256, 256), - 0, - (50, 5, 256, 256), - ), - pytest.param( - "image_stack_tpzc_50tp_2p_5z_3c_512k_1_MMStack_2-Pos000_000.ome.tif", - REMOTE, - 1, # Start with second scene to trigger error faster - (50, 5, 256, 256), - 0, - (50, 5, 256, 256), - marks=pytest.mark.raises(exceptions=IndexError), - ), - ], -) -@pytest.mark.parametrize("processes", [True, False]) -def test_parallel_multifile_tiff_read( - filename: str, - host: str, - first_scene: int, - expected_first_chunk_shape: Tuple[int, ...], - second_scene: int, - expected_second_chunk_shape: Tuple[int, ...], - processes: bool, -) -> None: - """ - This test ensures that we can serialize and read 'multi-file multi-scene' formats. - See: https://github.com/AllenCellModeling/aicsimageio/issues/196 - - We specifically test with a Distributed cluster to ensure that we serialize and - read properly from each file. - """ - # Construct full filepath - uri = get_resource_full_path(filename, host) - - # Init image - img = AICSImage(uri) - - # Init cluster - cluster = LocalCluster(processes=processes) - client = Client(cluster) - - # Select data - img.set_scene(first_scene) - first_out = img.get_image_dask_data("TZYX").compute() - assert first_out.shape == expected_first_chunk_shape - - # Update scene and select data - img.set_scene(second_scene) - second_out = img.get_image_dask_data("TZYX").compute() - assert second_out.shape == expected_second_chunk_shape - - # Shutdown and then safety measure timeout - cluster.close() - client.close() - time.sleep(5) + AICSImage(uri) diff --git a/aicsimageio/tests/utils/test_dask_proxy.py b/aicsimageio/tests/utils/test_dask_proxy.py deleted file mode 100644 index 22fc7d7bc..000000000 --- a/aicsimageio/tests/utils/test_dask_proxy.py +++ /dev/null @@ -1,113 +0,0 @@ -import warnings -from typing import Any, Tuple - -import dask.array as da -import numpy as np -import pytest - -from aicsimageio.utils.dask_proxy import DaskArrayProxy - - -# a *re-entrant* file context manager -class FileContext: - FILE_OPEN = False - OPEN_COUNT = 0 - - @property - def closed(self) -> bool: - return not self.FILE_OPEN - - def __enter__(self) -> "FileContext": - if not self.FILE_OPEN: - self.OPEN_COUNT += 1 - self.FILE_OPEN = True - return self - - def __exit__(self, *args: Any) -> None: - self.FILE_OPEN = False - - -@pytest.fixture -def dask_arr() -> da.Array: - called = [0] - ctx = FileContext() - - def get_chunk(block_id: Tuple[int, ...]) -> np.ndarray: - if not ctx.FILE_OPEN: - warnings.warn("You didn't open the file!") - nonlocal called - - if not isinstance(block_id, np.ndarray): - called[0] += 1 - return np.arange(100).reshape(10, 10)[np.newaxis, np.newaxis] - - d = da.map_blocks(get_chunk, chunks=((1,) * 10, (1,) * 10, 10, 10), dtype=float) - d.called = called - d.ctx = ctx - return d - - -@pytest.fixture -def proxy(dask_arr: da.Array) -> DaskArrayProxy: - return DaskArrayProxy(dask_arr, dask_arr.ctx) - - -def test_array(dask_arr: da.Array) -> None: - with pytest.warns(UserWarning): - dask_arr.compute() - - assert dask_arr.ctx.OPEN_COUNT == 0 - - with dask_arr.ctx: - assert dask_arr.compute().shape == (10, 10, 10, 10) - - assert dask_arr.ctx.OPEN_COUNT == 1 - - -def test_proxy_compute(proxy: DaskArrayProxy) -> None: - assert proxy.ctx.OPEN_COUNT == 0 - ary = proxy.compute() - assert isinstance(ary, np.ndarray) - assert ary.shape == (10, 10, 10, 10) - assert proxy.ctx.OPEN_COUNT == 1 - assert proxy.__wrapped__.called[0] == 100 - - -def test_proxy_asarray(proxy: DaskArrayProxy) -> None: - assert proxy.ctx.OPEN_COUNT == 0 - ary = np.asarray(proxy) - assert isinstance(ary, np.ndarray) - assert ary.shape == (10, 10, 10, 10) - assert proxy.ctx.OPEN_COUNT == 1 - assert proxy.__wrapped__.called[0] == 100 - - -def test_proxy_getitem(dask_arr: da.Array, proxy: DaskArrayProxy) -> None: - a = dask_arr[0, 1:3] - b = proxy[0, 1:3] - assert isinstance(a, da.Array) - assert isinstance(b, DaskArrayProxy) - np.testing.assert_array_equal(a.compute(), b.compute()) - - -def test_proxy_methods(dask_arr: da.Array, proxy: DaskArrayProxy) -> None: - dmean = proxy.mean() - assert isinstance(dmean, DaskArrayProxy) - assert isinstance(dmean.compute(), float) - assert dmean.compute() == dask_arr.mean().compute() - - # non array-returning methods don't return proxies - assert isinstance(proxy.to_svg(), str) - - -def test_proxy_ufunc(dask_arr: da.Array, proxy: DaskArrayProxy) -> None: - amean = np.mean(dask_arr) - pmean = np.mean(proxy) - assert isinstance(amean, da.Array) - assert isinstance(pmean, DaskArrayProxy) - assert amean.compute() == pmean.compute() - - -def test_proxy_repr(dask_arr: da.Array, proxy: DaskArrayProxy) -> None: - assert repr(dask_arr) == repr(proxy) - assert repr(dask_arr.mean) == repr(proxy.mean) diff --git a/aicsimageio/tests/writers/extra_writers/__init__.py b/aicsimageio/tests/writers/extra_writers/__init__.py new file mode 100644 index 000000000..faa18be5b --- /dev/null +++ b/aicsimageio/tests/writers/extra_writers/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- diff --git a/aicsimageio/tests/writers/test_timeseries_writer.py b/aicsimageio/tests/writers/extra_writers/test_timeseries_writer.py similarity index 98% rename from aicsimageio/tests/writers/test_timeseries_writer.py rename to aicsimageio/tests/writers/extra_writers/test_timeseries_writer.py index b971ee9ac..bf4e141cf 100644 --- a/aicsimageio/tests/writers/test_timeseries_writer.py +++ b/aicsimageio/tests/writers/extra_writers/test_timeseries_writer.py @@ -10,7 +10,7 @@ from aicsimageio.readers.default_reader import DefaultReader from aicsimageio.writers.timeseries_writer import TimeseriesWriter -from ..conftest import LOCAL, array_constructor, get_resource_write_full_path +from ...conftest import LOCAL, array_constructor, get_resource_write_full_path @array_constructor diff --git a/aicsimageio/tests/writers/test_two_d_writer.py b/aicsimageio/tests/writers/extra_writers/test_two_d_writer.py similarity index 96% rename from aicsimageio/tests/writers/test_two_d_writer.py rename to aicsimageio/tests/writers/extra_writers/test_two_d_writer.py index 0487c7399..a7e43335a 100644 --- a/aicsimageio/tests/writers/test_two_d_writer.py +++ b/aicsimageio/tests/writers/extra_writers/test_two_d_writer.py @@ -10,7 +10,7 @@ from aicsimageio.readers.default_reader import DefaultReader from aicsimageio.writers.two_d_writer import TwoDWriter -from ..conftest import LOCAL, array_constructor, get_resource_write_full_path +from ...conftest import LOCAL, array_constructor, get_resource_write_full_path @array_constructor diff --git a/aicsimageio/utils/cached_property.py b/aicsimageio/utils/cached_property.py deleted file mode 100644 index 746c0d5c5..000000000 --- a/aicsimageio/utils/cached_property.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -back-ported cached_property from the standard library (functools) in python 3.8. - -TODO: remove when we drop support for python 3.7 - -It transforms a method of a class into a property whose value is computed once and then -cached as a normal attribute for the life of the instance. Similar to property(), with -the addition of caching. Useful for expensive computed properties of instances that are -otherwise effectively immutable. - -""" -from typing import Any, Callable, Optional - -try: - from functools import cached_property -except ImportError: - from threading import RLock - - _NOT_FOUND = object() - - class cached_property: # type: ignore # noqa - def __init__(self, func: Callable): - self.func = func - self.attrname: Optional[str] = None - self.__doc__ = func.__doc__ - self.lock = RLock() - - def __set_name__(self, owner: object, name: str) -> None: - if self.attrname is None: - self.attrname = name - elif name != self.attrname: - raise TypeError( - "Cannot assign the same cached_property to two different names " - f"({self.attrname!r} and {name!r})." - ) - - def __get__(self, instance: object, owner: type = None) -> Any: - if instance is None: - return self - if self.attrname is None: - raise TypeError( - "Cannot use cached_property instance without calling " - "__set_name__ on it." - ) - try: - # not all objects have __dict__ (e.g. class defines slots) - cache = instance.__dict__ - except AttributeError: - msg = ( - f"No '__dict__' attribute on {type(instance).__name__!r} " - f"instance to cache {self.attrname!r} property." - ) - raise TypeError(msg) from None - val = cache.get(self.attrname, _NOT_FOUND) - if val is _NOT_FOUND: - with self.lock: - # check if another thread filled cache while we awaited lock - val = cache.get(self.attrname, _NOT_FOUND) - if val is _NOT_FOUND: - val = self.func(instance) - try: - cache[self.attrname] = val - except TypeError: - name = type(instance).__name__ - msg = ( - f"The '__dict__' attribute on {name!r} " - "instance does not support item assignment for caching " - f"{self.attrname!r} property." - ) - raise TypeError(msg) from None - return val diff --git a/aicsimageio/utils/dask_proxy.py b/aicsimageio/utils/dask_proxy.py deleted file mode 100644 index 0b7bb65ba..000000000 --- a/aicsimageio/utils/dask_proxy.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -experimental Dask array proxy for file IO -""" -from __future__ import annotations - -from contextlib import nullcontext -from typing import TYPE_CHECKING, Any, Callable - -import dask.array as da -import numpy as np -from wrapt import ObjectProxy - -if TYPE_CHECKING: - from typing_extensions import Protocol - - # fmt: off - class CheckableContext(Protocol): - @property - def closed(self) -> bool: ... # noqa: E704 - def __enter__(self) -> CheckableContext: ... # noqa: E704 - def __exit__(self, *a: Any) -> None: ... # noqa: E704 - # fmt: on - - -class DaskArrayProxy(ObjectProxy): - """Dask array wrapper that provides a 'file open' context when computing. - - This is necessary when the dask array contains a delayed underlying reader function - that requires the file to be open. We don't want to open/close the file on - every single chunk. So this wrap the `compute` and `__array__` method in an - open-file context manager. For __getattr__, we return an `ArrayMethodProxy` that - again wraps the resulting object in a DaskArrayProxy if it is a dask array. - - Experimental! - The state held by the `file_ctx` may be problematic for dask distributed. - - Parameters - ---------- - wrapped : da.Array - the dask array that requires some file - file_ctx : ContextManager - A context in which the file is open. - IMPORTANT: the context must be reusable, and preferably re-entrant: - https://docs.python.org/3/library/contextlib.html#reentrant-context-managers - """ - - __wrapped__: da.Array - - def __init__(self, wrapped: da.Array, file_ctx: CheckableContext) -> None: - super().__init__(wrapped) - self._file_ctx = file_ctx - - def __getitem__(self, key: Any) -> DaskArrayProxy: - return DaskArrayProxy(self.__wrapped__.__getitem__(key), self._file_ctx) - - def __getattr__(self, key: Any) -> Any: - attr = getattr(self.__wrapped__, key) - return _ArrayMethodProxy(attr, self._file_ctx) if callable(attr) else attr - - def __repr__(self) -> str: - return repr(self.__wrapped__) - - def compute(self, **kwargs: Any) -> np.ndarray: - with self._file_ctx if self._file_ctx.closed else nullcontext(): # type: ignore - return self.__wrapped__.compute(**kwargs) - - def __array__(self, dtype: str = None, **kwargs: Any) -> np.ndarray: - with self._file_ctx if self._file_ctx.closed else nullcontext(): # type: ignore - return self.__wrapped__.__array__(dtype, **kwargs) - - -class _ArrayMethodProxy: - """Wraps method on a dask array and returns a DaskArrayProxy if the result of the - method is a dask array. see details in DaskArrayProxy docstring.""" - - def __init__(self, method: Callable, file_ctx: CheckableContext) -> None: - self.method = method - self._file_ctx = file_ctx - - def __repr__(self) -> str: - return repr(self.method) - - def __call__(self, *args: Any, **kwds: Any) -> Any: - with self._file_ctx if self._file_ctx.closed else nullcontext(): # type: ignore - result = self.method(*args, **kwds) - - if isinstance(result, da.Array): - return DaskArrayProxy(result, self._file_ctx) - return result diff --git a/asv.conf.json b/asv.conf.json index 4d220fc5c..80f26a91e 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -10,7 +10,7 @@ "in-dir={env_dir} python -mpip install {build_dir}[benchmark]" ], "show_commit_url": "https://github.com/AllenCellModeling/aicsimageio/commit/", - "pythons": ["3.9"], + "pythons": ["3.10"], "benchmark_dir": "benchmarks", "env_dir": ".asv/env", "results_dir": ".asv/results", diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a6c207261..3a11311e4 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -49,6 +49,18 @@ Ready to contribute? Here's how to set up `aicsimageio` for local development. make build-with-remote + If you want to test just core readers and writers (Tiff and OmeTiff) run core tests: + + tox -e py + + If you want to test your custom reader or writer, be sure to add a tox configuration + and the name of the custom tox env to the matrix on our GitHub Actions. Once added, + the testing pattern should follow: + + tox -e lif + tox -e bfio + tox -e czi + 8. (Optional) If you'd like to have linting checks run automatically prior to every `git commit`, then you can install a commit-hook that runs [pre-commit](https://pre-commit.com/), as follows: @@ -65,13 +77,17 @@ Ready to contribute? Here's how to set up `aicsimageio` for local development. 10. Submit a pull request through the GitHub website. ## Updating the Submodule + If there are updates to the submodule repo that you would like to integrate into AICSImageIO, follow these steps: + 1. Create a new branch in this repo: + ``` git checkout -b {your_development_type}/short-description ``` 2. Pull in the changes from the submodule: + ``` git submodule update --remote --merge ``` diff --git a/setup.py b/setup.py index f33af68a6..2e6031f82 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def run(self): ], "nd2": ["nd2[legacy]>=0.2.0"], "dv": ["mrc>=0.2.0"], - "bfio": ["bfio>=2.3.0"], + "bfio": ["bfio>=2.3.0", "tifffile<2022.4.22"], # "czi": ["aicspylibczi>=3.0.5"], # excluded for licensing reasons # "bioformats": ["bioformats_jar"], # excluded for licensing reasons # "lif": ["readlif>=0.6.4"], # excluded for licensing reasons @@ -52,7 +52,6 @@ def run(self): ] test_requirements = [ - *all_formats, "codecov>=2.1.4", "distributed>=2021.4.1", "docutils>=0.10,<0.16", @@ -63,10 +62,6 @@ def run(self): "quilt3", # no pin to avoid pip cycling (boto is really hard to manage) "s3fs[boto3]>=0.4.2", "tox>=3.15.2", - "bioformats_jar", # to test bioformats - "bfio>=2.3.0", - "readlif>=0.6.4", # to test lif - "aicspylibczi>=3.0.5", # to test czi ] dev_requirements = [ @@ -88,6 +83,12 @@ def run(self): "sphinx_rtd_theme>=0.5.1", "twine>=3.1.1", "wheel>=0.34.2", + # reader deps + *all_formats, + "bioformats_jar", # to test bioformats + "bfio>=2.3.0", + "readlif>=0.6.4", # to test lif + "aicspylibczi>=3.0.5", # to test czi ] benchmark_requirements = [ @@ -103,6 +104,7 @@ def run(self): "numpy>=1.16,<2", "ome-types>=0.2", "wrapt>=1.12", + "resource-backed-dask-array>=0.1.0", "tifffile>=2021.8.30", "xarray>=0.16.1", "xmlschema", # no pin because it's pulled in from OME types @@ -129,9 +131,9 @@ def run(self): "Intended Audience :: Education", "License :: OSI Approved :: BSD License", "Natural Language :: English", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], description=( "Image Reading, Metadata Conversion, and Image Writing for Microscopy Images " @@ -155,7 +157,7 @@ def run(self): "*.benchmarks.*", ] ), - python_requires=">=3.7", + python_requires=">=3.8", setup_requires=setup_requirements, test_suite="aicsimageio/tests", tests_require=test_requirements, diff --git a/tox.ini b/tox.ini index 088a156dc..b2458a9ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, lint +envlist = py38, py39, py310, bioformats, czi, base-imageio, dv, lif, nd2, bfio, lint skip_missing_interpreters = true toxworkdir={env:TOX_WORK_DIR:.tox} @@ -8,6 +8,97 @@ skip_install = true deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure +[testenv:bioformats] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test +deps = bioformats_jar +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py {posargs} + +[testenv:czi] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test +deps = aicspylibczi>=3.0.5 +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_czi_reader.py {posargs} + +[testenv:base-imageio] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test + base-imageio +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_default_reader.py aicsimageio/tests/writers/extra_writers/test_timeseries_writer.py aicsimageio/tests/writers/extra_writers/test_two_d_writer.py {posargs} + +[testenv:dv] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test + dv +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_dv_reader.py {posargs} + +[testenv:lif] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test +deps = readlif>=0.6.4 +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_lif_reader.py {posargs} + +[testenv:nd2] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test + nd2 +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_nd2_reader.py {posargs} + +[testenv:bfio] +passenv = + AWS_* + JAVA_HOME + CI +setenv = + PYTHONPATH = {toxinidir} +extras = + test + bfio +commands = + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py {posargs} + [testenv] passenv = AWS_* @@ -18,4 +109,4 @@ setenv = extras = test commands = - pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/ {posargs} + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/ --ignore-glob="**/extra_readers/*" --ignore-glob="**/extra_writers/*" {posargs}