diff --git a/.github/issue_template/bug_report.md b/.github/issue_template/bug_report.md
index 6d6f41234..ca7033a42 100644
--- a/.github/issue_template/bug_report.md
+++ b/.github/issue_template/bug_report.md
@@ -1,18 +1,35 @@
-**Description:**
-A clear and concise description of what the bug is.
+---
+name: Bug Report
+about: '"Something''s wrong..."'
+title: ''
+labels: ''
+assignees: ''
-**Expected Behavior:**
-A clear and concise description of what you expected to happen.
+---
-**System and Software:**
+## System and Software
* aicsimageio Version:
* Python Version:
* Operating System:
-**Steps to Reproduce:**
-```
-Paste the command(s) you used and the output.
-```
- **Additional context**
-Add any other context about the problem here.
+## Description
+*A clear description of the bug*
+
+
+
+
+## Expected Behavior
+*What did you expect to happen instead?*
+
+
+
+
+## Reproduction
+*A minimal example that exhibits the behavior.*
+
+
+
+
+## Environment
+*Any additional information about your environment*
diff --git a/.github/issue_template/feature_request.md b/.github/issue_template/feature_request.md
index fbfb7b012..3b0421556 100644
--- a/.github/issue_template/feature_request.md
+++ b/.github/issue_template/feature_request.md
@@ -1,5 +1,23 @@
-**Description:**
-A clear and concise description of what you want to happen.
+---
+name: Feature Request
+about: '"It would be really cool if x did y..."'
+title: ''
+labels: ''
+assignees: ''
-**Additional context**
-Add any other context or screenshots about the feature request here.
+---
+
+## Use Case
+*Please provide a use case to help us understand your request in context*
+
+
+
+
+## Solution
+*Please describe your ideal solution*
+
+
+
+
+## Alternatives
+*Please describe any alternatives you've considered, even if you've dismissed them*
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
index b1d7c9425..2b4e18939 100644
--- a/.github/workflows/build-docs.yml
+++ b/.github/workflows/build-docs.yml
@@ -1,4 +1,4 @@
-name: Docs
+name: Documentation
on:
push:
@@ -9,12 +9,23 @@ jobs:
docs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@master
+ - uses: actions/checkout@v1
+ - name: Set up Python
+ uses: actions/setup-python@v1
+ with:
+ python-version: 3.7
+ - name: Install Dependencies
+ run: |
+ pip install --upgrade pip
+ pip install .[all]
- name: Generate Docs
- uses: JacksonMaxfield/github-pages-deploy-action-python@master
- env:
+ run: |
+ make gen-docs
+ touch docs/_build/html/.nojekyll
+ - name: Publish Docs
+ uses: JamesIves/github-pages-deploy-action@releases/v3
+ with:
ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
BASE_BRANCH: master # The branch the action should deploy from.
BRANCH: gh-pages # The branch the action should deploy to.
- FOLDER: docs/_build/html # The folder the action should deploy. This example folder is generated by Sphinx
- BUILD_SCRIPT: pip install .[dev] && make docs && touch docs/_build/html/.nojekyll # The build script the action should run prior to deploying.
+ FOLDER: docs/_build/html/ # The folder the action should deploy.
diff --git a/.github/workflows/build-master.yml b/.github/workflows/build-master.yml
index ba3db107d..567fec71a 100644
--- a/.github/workflows/build-master.yml
+++ b/.github/workflows/build-master.yml
@@ -20,12 +20,12 @@ jobs:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- - uses: actions/checkout@master
+ - uses: actions/checkout@v1
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@master
+ uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- - name: Install dependencies
+ - name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install .[test]
@@ -38,12 +38,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@master
+ - uses: actions/checkout@v1
- name: Set up Python 3.7
- uses: actions/setup-python@master
+ uses: actions/setup-python@v1
with:
python-version: 3.7
- - name: Install dependencies
+ - name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install .[test]
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 6f8104458..f9081c3f6 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,20 +9,20 @@ jobs:
publish:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@master
+ - uses: actions/checkout@v1
- name: Set up Python
- uses: actions/setup-python@master
+ uses: actions/setup-python@v1
with:
python-version: 3.7
- - name: Install dependencies
+ - name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel
- - name: Build package
+ - name: Build Package
run: |
python setup.py sdist bdist_wheel
- name: Publish to PyPI
- uses: pypa/gh-action-pypi-publish@master
+ uses: pypa/gh-action-pypi-publish@v1
with:
user: aicspypi
password: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml
index 48325c529..28d3f368e 100644
--- a/.github/workflows/test-and-lint.yml
+++ b/.github/workflows/test-and-lint.yml
@@ -12,30 +12,29 @@ jobs:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- - uses: actions/checkout@master
+ - uses: actions/checkout@v1
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@master
+ uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- - name: Install dependencies
+ - name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install .[test]
- name: Test with pytest
run: |
pytest --cov-report xml --cov=aicsimageio aicsimageio/tests/
- codecov -t ${{ secrets.CODECOV_TOKEN }}
lint:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@master
+ - uses: actions/checkout@v1
- name: Set up Python 3.7
- uses: actions/setup-python@master
+ uses: actions/setup-python@v1
with:
python-version: 3.7
- - name: Install dependencies
+ - name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install .[test]
diff --git a/.gitignore b/.gitignore
index 45e655d1e..87c594cb8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -69,6 +69,9 @@ instance/
docs/_build/
docs/aicsimageio.*rst
+# Dask workers
+dask-worker-space/
+
# PyBuilder
target/
diff --git a/README.md b/README.md
index ae9d2b644..ca79a11a5 100644
--- a/README.md
+++ b/README.md
@@ -9,69 +9,128 @@ A Python library for reading and writing image data with specific support for ha
---
## Features
-* Supports reading metadata and imaging data from file path or buffered bytes for:
+* Supports reading metadata and imaging data for:
* `CZI`
* `OME-TIFF`
* `TIFF`
- * Any additional format supported by `imageio`
+ * Any additional format supported by [`imageio`](https://github.com/imageio/imageio)
* Supports writing metadata and imaging data for:
* `OME-TIFF`
- * `TIFF`
- * Any additional format supported by `imageio`
-### Disclaimer:
-This package is under heavy revision in preparation for version 3.0.0 release. The quick start below is representative
-of how to interact with the package under 3.0.0 and not under the current stable release.
+## Installation
+**Stable Release:** `pip install aicsimageio`
+**Development Head:** `pip install git+https://github.com/AllenCellModeling/aicsimageio.git`
+
+## Documentation
+For full package documentation please visit
+[allencellmodeling.github.io/aicsimageio](https://allencellmodeling.github.io/aicsimageio/index.html).
## Quick Start
+
+### Full Image Reading
```python
from aicsimageio import AICSImage, imread
-# For numpy array of image data
-im = imread("/path/to/your/file_or_buffer.ome.tiff")
+# Get an AICSImage object
+img = AICSImage("my_file.tiff")
+img.data # returns 6D STCZYX numpy array
+img.dims # returns string "STCZYX"
+img.shape # returns tuple of dimension sizes in STCZYX order
+img.size("STC") # returns tuple of dimensions sizes for just STC
+img.get_image_data("CZYX", S=0, T=0) # returns 4D CZYX numpy array
-# For AICSImage object that
-with AICSImage("/path/to/your/file_or_buffer.ome.tiff") as im:
- # use im object
+# Get 6D STCZYX numpy array
+data = imread("my_file.tiff")
+```
-# To specify a known dimension order
-with AICSImage("/path/to/your/file_or_buffer.ome.tiff", known_dims="SYX") as im:
- # use im object
+### Delayed Image Slice Reading
+```python
+from aicsimageio import AICSImage, imread_dask
+
+# Get an AICSImage object
+img = AICSImage("my_file.tiff")
+img.dask_data # returns 6D STCZYX dask array
+img.dims # returns string "STCZYX"
+img.shape # returns tuple of dimension sizes in STCZYX order
+img.size("STC") # returns tuple of dimensions sizes for just STC
+img.get_image_dask_data("CZYX", S=0, T=0) # returns 4D CZYX dask array
+
+# Read specified portion of dask array
+lazy_s0t0 = img.get_image_dask_data("CZYX", S=0, T=0) # returns 4D CZYX dask array
+s0t0 = lazy_s0t0.compute() # returns 4D CZYX numpy array
+
+# Or use normal numpy array slicing
+lazy_data = imread_dask("my_file.tiff")
+lazy_s0t0 = lazy_data[0, 0, :]
+s0t0 = lazy_s0t0.compute()
+```
-# if you instantiate an AICSImage:
-im = AICSImage("/path/to/your/file_or_buffer.ome.tiff")
-# you should close it when done:
-im.close()
+When using the `dask_data` array, it is important to know when to `compute` or `persist` data and when to keep
+chaining computation. [Here is a good rundown on the trade offs.](https://stackoverflow.com/questions/41806850/dask-difference-between-client-persist-and-client-compute#answer-41807160)
-# Image data is stored in `data` attribute
-im.data # returns the image data numpy array
-# Image dimension sizes can be obtained via properties:
-im.size_z # returns the size of the Z dimension. X,Y,Z,C,T, and S supported.
+### Speed up IO and Processing with Dask Clients and Clusters
+```python
+from aicsimageio import AICSImage, dask_utils
+
+# Create a local dask cluster and client for the duration of the context manager
+with AICSImage("filename.ome.tiff") as img:
+ # do your work like normal
+ print(img.dask_data.shape)
+
+# Specify arguments for the local cluster initialization
+with AICSImage("filename.ome.tiff", dask_kwargs={"nworkers": 4}) as img:
+ # do your work like normal
+ print(img.dask_data.shape)
+
+# Connect to a dask client for the duration of the context manager
+with AICSImage("filename.ome.tiff", dask_kwargs={"address": "tcp://localhost:12345"}) as img:
+ # do your work like normal
+ print(img.dask_data.shape)
+
+# Or spawn a local cluster and / or connect to a client outside of a context manager
+# This uses the same "address" and dask kwargs as above
+# If you pass an address in, it will create and shutdown the client and no cluster will be created.
+# Similar to AICSImage, these objects will be connected and useable for the lifespan of the context manager.
+with dask_utils.cluster_and_client() as (cluster, client):
+
+ img1 = AICSImage("1.tiff")
+ img2 = AICSImage("2.tiff")
+ img3 = AICSImage("3.tiff")
+
+ # Do your image processing work
+```
-# Image dimensions can also be obtained as a tuple in two ways:
-im.size("ZYX") # returns a tuple containing the Z, Y, and X sizes only
-im.get_image_data(out_orientation="ZYX").shape # returns same as above
+### Metadata Reading
+```python
+from aicsimageio import AICSImage
-# Image metadata is stored in `metadata` attribute
-im.metadata # returns whichever metadata parser best suits the file format
+# Get an AICSImage object
+img = AICSImage("my_file.tiff")
+img.metadata # returns the metadata object for this image type
+img.get_channel_names() # returns a list of string channel names if found in the metadata
+```
-# Subsets or transposes of the image data can be requested:
-im.get_image_data(out_orientation="ZYX") # returns a 3d data block containing only the ZYX dimensions
+### Napari Interactive Viewer
+[napari](https://github.com/Napari/napari) is a fast, interactive, multi-dimensional image viewer for python and
+it is pretty useful for imaging data that this package tends to interact with.
+```python
+from aicsimageio import AICSImage
+# Get an AICSImage object
+img = AICSImage("my_file.tiff")
+img.view_napari() # launches napari GUI and viewer
```
## Notes
-* Image data numpy arrays are always returned as six dimensional in dimension order `STCZYX`
+* Image `data` and `dask_data` are always returned as six dimensional in dimension order `STCZYX`
or `Scene`, `Time`, `Channel`, `Z`, `Y`, and `X`.
* Each file format may use a different metadata parser it is dependent on the reader's implementation.
-
-## Installation
-**Stable Release:** `pip install aicsimageio`
-**Development Head:** `pip install git+https://github.com/AllenCellModeling/aicsimageio.git`
-
-## Documentation
-For full package documentation please visit [allencellmodeling.github.io/aicsimageio](https://allencellmodeling.github.io/aicsimageio/index.html).
+* The `AICSImage` object will only pull the `Scene`, `Time`, `Channel`, `Z`, `Y`, `X` dimensions from the reader.
+If your file has dimensions outside of those, use the base reader classes `CziReader`, `OmeTiffReader`, `TiffReader`,
+or `DefaultReader`.
+* We make some choices for the user based off the image data during `img.view_napari`.
+If you don't want this behavior, simply pass the `img.dask_data` into `napari.view_image` instead.
## Development
See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the code.
diff --git a/aicsimageio/__init__.py b/aicsimageio/__init__.py
index 9f1672fe0..97aec7025 100644
--- a/aicsimageio/__init__.py
+++ b/aicsimageio/__init__.py
@@ -1,5 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
from .aics_image import AICSImage # noqa: F401
from .aics_image import imread # noqa: F401
+from .aics_image import imread_dask # noqa: F401
# Do not edit this string manually, always use bumpversion
# Details in CONTRIBUTING.md
diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py
index 8d5899b8a..33947fdef 100644
--- a/aicsimageio/aics_image.py
+++ b/aicsimageio/aics_image.py
@@ -1,242 +1,280 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import logging
-import typing
-from typing import Optional, Type
+from typing import Any, Dict, List, Optional, Tuple, Type
+import dask.array as da
import numpy as np
-
-from . import constants, transforms, types
-from .exceptions import InvalidDimensionOrderingError, UnsupportedFileFormatError
-from .readers import CziReader, DefaultReader, NdArrayReader, OmeTiffReader, TiffReader
+from distributed import Client, LocalCluster
+
+from . import dask_utils, transforms, types
+from .constants import Dimensions
+from .exceptions import (InvalidDimensionOrderingError,
+ UnsupportedFileFormatError)
+from .readers import (ArrayLikeReader, CziReader, DefaultReader, OmeTiffReader,
+ TiffReader)
from .readers.reader import Reader
+###############################################################################
+
log = logging.getLogger(__name__)
+###############################################################################
+
+# The order of the readers in this list is important.
+# Example: if TiffReader was placed before OmeTiffReader, we would never hit the OmeTiffReader.
+SUPPORTED_READERS = [ArrayLikeReader, CziReader, OmeTiffReader, TiffReader, DefaultReader]
+
+###############################################################################
+
class AICSImage:
"""
- AICSImage takes microscopy image data types (files / bytestreams) of varying dimensions ("ZYX", "TCZYX", "CYX") and
- puts them into a consistent 6D "STCZYX" ordered numpy.ndarray. The data, metadata are lazy loaded and can be
+ AICSImage takes microscopy image data types (files) of varying dimensions ("ZYX", "TCZYX", "CYX") and
+ puts them into a consistent 6D "STCZYX" ordered dask array. The data, metadata are lazy loaded and can be
accessed as needed. Note the dims are assumed to match "STCZYX" from right to left meaning if 4 dimensional
data is provided then the dimensions are assigned to be "CZYX", 2 dimensional would be "YX". This guessed assignment
- is only for file types without dimension metadata (ie not .ome.tiff or .czi).
-
- Note: if you absolutely know the dims and they are not as guessed then you can override them by immediately setting
- them after initialization see example below.
+ is only for file types without dimension metadata (i.e. not .ome.tiff or .czi).
Simple Example
--------------
- with open("filename.czi", 'rb') as fp:
- img = AICSImage(fp)
- data = img.data # data is a 6D "STCZYX" object
- metadata = img.metadata # metadata from the file, an xml.etree
- zstack_t8 = img.get_image_data("ZYX", S=0, T=8, C=0) # returns a 3D "ZYX" numpy.ndarray
-
- zstack_t10 = data[0, 10, 0, :, :, :] # access the S=0, T=10, C=0 "ZYX" cube
+ img = AICSImage("my_file.tiff")
+ data = img.data # data is a 6D "STCZYX" dask array
+ metadata = img.metadata # metadata from the file, an xml.etree
+ zstack_t8 = img.get_image_data("ZYX", S=0, T=8, C=0) # returns a 3D "ZYX" numpy array
+ zstack_t10 = data[0, 10, 0, :, :, :] # access the S=0, T=10, C=0 "ZYX" dask array
+ zstack_t10_np = zstack_t10.compute() # read the data from the file and convert from dask array to numpy array
File Examples
-------------
-
- with AICSImage("filename.ome.tif") as img:
- # do work with img
- data = img.data
- metadata = img.metadata
- # img will be closed automatically at end of scope
-
- OmeTif
- img = AICSImage("filename.ome.tif")
- img.close()
+ OME-TIFF
+ img = AICSImage("filename.ome.tiff")
CZI (Zeiss)
- img = AICSImage("filename.czi") or AICSImage("filename.czi", max_workers=8)
- img.close()
- Tiff
- img = AICSImage("filename.tif")
- img.close()
- Png/Gif/...
+ img = AICSImage("filename.czi")
+ TIFF
+ img = AICSImage("filename.tiff")
+ PNG / GIF / ...
img = AICSImage("filename.png")
- img.close()
img = AICSImage("filename.gif")
- img.close()
-
- Bytestream Examples
- -------------------
- OmeTif
- with open("filename.ome.tif", 'rb') as fp:
- img = AICSImage(fp)
- CZI
- with open("filename.czi", 'rb') as fp:
- img = AICSImage(fp, max_workers=7)
- Tiff/Png/Gif
- with open("filename.png", 'rb') as fp:
- img = AICSImage(fp)
-
- Numpy.ndarray Example
+
+ dask.array.Array Example
+ ------------------------
+ blank = dask.array.zeros((2, 600, 900))
+ img = AICSImage(blank)
+
+ numpy.ndarray Example
---------------------
blank = numpy.zeros((2, 600, 900))
img = AICSImage(blank)
- Example with known dimensions different than guessed
- ----------------------------------------------------
- img = AICSImage('TCXimage.gif')
- img.reader.dims = 'TCX'
- data = img.data # get a 6D ndarray back in "STCZYX" order
- """
+ Dask Client / Cluster Example
+ -----------------------------
+ # Create a local dask cluster for the duration of the context manager
+ with AICSImage("filename.ome.tiff") as img:
+ # do your work like normal
- SUPPORTED_READERS = [CziReader, OmeTiffReader, TiffReader, DefaultReader]
+ # Specify arguments for the local cluster initialization
+ with AICSImage("filename.ome.tiff", dask_kwargs={"nworkers": 4}) as img:
+ # do your work like normal
+
+ # Connect to a dask client for the duration of the context manager
+ with AICSImage("filename.ome.tiff", dask_kwargs={"address": "tcp://localhost:12345"}) as img:
+ # do your work like normal
+ """
def __init__(
self,
- data: typing.Union[types.FileLike, np.ndarray],
+ data: types.ImageLike,
known_dims: Optional[str] = None,
- **kwargs,
+ dask_kwargs: Dict[str, Any] = {},
+ **kwargs
):
"""
Constructor for AICSImage class intended for providing a unified interface for dealing with
microscopy images. To extend support to a new reader simply add a new reader child class of
- Reader ([readers/reader.py]) and add the class to SUPPORTED_READERS in AICSImage.
+ Reader ([readers/reader.py]) and add the class to SUPPORTED_READERS variable.
Parameters
----------
- data: String with path to ometif/czi/tif/png/gif file, or ndarray with up to 6 dimensions
- known_dims: Optional string with the known dimension order. If None, the reader will attempt to parse dim order.
- kwargs: Parameters to be passed through to the reader class
- max_workers (optional Czi) specifies the number of worker threads for the backend library
+ data: types.ImageLike
+ String with path to file, numpy.ndarray, or dask.array.Array with up to six dimensions.
+ known_dims: Optional[str]
+ Optional string with the known dimension order. If None, the reader will attempt to parse dim order.
+ dask_kwargs: Dict[str, Any] = {}
+ A dictionary of arguments to pass to a dask cluster and or client.
+ kwargs: Dict[str, Any]
+ Extra keyword arguments that can be passed down to either the reader subclass or, if using the context
+ manager, the LocalCluster initialization.
"""
+ # Check known dims
+ if known_dims is not None:
+ if not all([d in Dimensions.DefaultOrder for d in known_dims]):
+ raise InvalidDimensionOrderingError(
+ f"The provided dimension string to the 'known_dims' argument includes dimensions that AICSImage "
+ f"does not support. Received: '{known_dims}'. Supported dimensions: {Dimensions.DefaultOrderList}."
+ )
# Hold onto known dims until data is requested
self._known_dims = known_dims
# Dims should nearly always be default dim order unless explictly overridden
- self.dims = constants.DEFAULT_DIMENSION_ORDER
+ self.dims = Dimensions.DefaultOrder
- # Lazy load later
+ # Determine reader class and create dask delayed array
+ reader_class = self.determine_reader(data=data)
+ self._reader = reader_class(data, **kwargs)
+
+ # Lazy load data from reader and reformat to standard dimensions
+ self._dask_data = None
self._data = None
self._metadata = None
- # Determine reader class and load data
- reader_class = self.determine_reader(data)
- self._reader = reader_class(data, **kwargs)
+ # Store dask client and cluster setup
+ self._dask_kwargs = dask_kwargs
+ self._client = None
+ self._cluster = None
@staticmethod
def determine_reader(data: types.ImageLike) -> Type[Reader]:
- """Cheaply check to see if a given file is a recognized type.
- Currently recognized types are TIFF, OME TIFF, and CZI.
- If the file is a TIFF, then the description (OME XML if it is OME TIFF) can be retrieved via read_description.
- Similarly, if the file is a CZI, then the metadata XML can be retrieved via read_description.
- """
- # The order of the readers in this list is important.
- # Example: if TiffReader was placed before OmeTiffReader, we would never use the OmeTiffReader.
- for reader_class in [
- NdArrayReader,
- CziReader,
- OmeTiffReader,
- TiffReader,
- DefaultReader,
- ]:
+ """
+ Cheaply check to see if a given file is a recognized type and return the appropriate reader for the file.
+ """
+ # Iterate through the ordered supported readers to find the right one
+ for reader_class in SUPPORTED_READERS:
if reader_class.is_this_type(data):
return reader_class
- raise UnsupportedFileFormatError(type(data))
+ raise UnsupportedFileFormatError(data)
@property
- def data(self):
+ def dask_data(self) -> da.core.Array:
"""
Returns
-------
- Returns a numpy.ndarray with dimension ordering "STCZYX"
+ Returns a dask array with dimension ordering "STCZYX"
"""
- if self._data is None:
- reader_data = self._reader.data
+ # Construct dask array if never before constructed
+ if self._dask_data is None:
+ reader_data = self.reader.dask_data
# Read and reshape and handle delayed known dims reshape
- self._data = transforms.reshape_data(
+ self._dask_data = transforms.reshape_data(
data=reader_data,
given_dims=self._known_dims or self.reader.dims,
return_dims=self.dims,
)
+
+ return self._dask_data
+
+ @property
+ def data(self) -> np.ndarray:
+ if self._data is None:
+ self._data = self.dask_data.compute()
+
return self._data
- def size(self, dims: str = "STCZYX"):
+ def size(self, dims: str = Dimensions.DefaultOrder) -> Tuple[int]:
"""
Parameters
----------
- dims: A string containing a list of dimensions being requested. The default is to return the 6 standard dims
+ dims: str
+ A string containing a list of dimensions being requested. The default is to return the six standard dims.
Returns
-------
Returns a tuple with the requested dimensions filled in
"""
+ # Ensure dims is an uppercase string
dims = dims.upper()
- if not (all(d in "STCZYX" for d in dims)):
+
+ # Check that dims requested are all a part of the available dims in the package
+ if not (all(d in Dimensions.DefaultOrder for d in dims)):
raise InvalidDimensionOrderingError(f"Invalid dimensions requested: {dims}")
+
+ # Check that the dims requested are in the image dims
if not (all(d in self.dims for d in dims)):
raise InvalidDimensionOrderingError(f"Invalid dimensions requested: {dims}")
- return tuple([self.data.shape[self.dims.index(dim)] for dim in dims])
+
+ # Return the shape of the data for the dimensions requested
+ return tuple([self.dask_data.shape[self.dims.index(dim)] for dim in dims])
+
+ @property
+ def shape(self) -> Tuple[int]:
+ return self.size()
@property
- def size_x(self):
+ def size_x(self) -> int:
"""
Returns
-------
- Returns the x size
+ size: int
+ The size of the Spatial X dimension.
"""
- return self.size("X")[0]
+ return self.size(Dimensions.SpatialX)[0]
@property
- def size_y(self):
+ def size_y(self) -> int:
"""
Returns
-------
- Returns the y size
+ size: int
+ The size of the Spatial Y dimension.
"""
- return self.size("Y")[0]
+ return self.size(Dimensions.SpatialY)[0]
@property
- def size_z(self):
+ def size_z(self) -> int:
"""
Returns
-------
- Returns the z size
+ size: int
+ The size of the Spatial Z dimension.
"""
- return self.size("Z")[0]
+ return self.size(Dimensions.SpatialZ)[0]
@property
- def size_c(self):
+ def size_c(self) -> int:
"""
Returns
-------
- Returns the c size
+ size: int
+ The size of the Channel dimension.
"""
- return self.size("C")[0]
+ return self.size(Dimensions.Channel)[0]
@property
- def size_t(self):
+ def size_t(self) -> int:
"""
Returns
-------
- Returns the t size
+ size: int
+ The size of the Time dimension.
"""
- return self.size("T")[0]
+ return self.size(Dimensions.Time)[0]
@property
- def size_s(self):
+ def size_s(self) -> int:
"""
Returns
-------
- Returns the s size
+ size: int
+ The size of the Scene dimension.
"""
- return self.size("S")[0]
+ return self.size(Dimensions.Scene)[0]
@property
- def metadata(self):
+ def metadata(self) -> Any:
"""
Returns
-------
- The Metadata from the Czi, or Ome.Tiff file, or other base class type with metadata.
- For pure image files and empty string or None is returned.
-
+ metadata: Any
+ The Metadata from the Czi, or Ome.Tiff file, or other base class type with metadata.
+ For pure image files an empty string or None is returned.
"""
if self._metadata is None:
- self._metadata = self._reader.metadata
+ self._metadata = self.reader.metadata
+
return self._metadata
@property
@@ -248,20 +286,20 @@ def reader(self) -> Reader:
Returns
-------
- A child of Reader, CziReader OmeTiffReader, TiffReader, DefaultReader, etc.
-
+ reader: Reader
+ A child of Reader; CziReader OmeTiffReader, TiffReader, DefaultReader, etc.
"""
return self._reader
- def get_image_data(
- self, out_orientation: str = None, copy: bool = False, **kwargs
- ) -> np.ndarray:
+ def get_image_dask_data(self, out_orientation: Optional[str] = None, **kwargs) -> da.core.Array:
"""
+ Get specific dimension image data out of an image as a dask array.
Parameters
----------
- out_orientation: A string containing the dimension ordering desired for the returned ndarray
- copy: boolean value to get image data by reference or by value [True, False]
+ out_orientation: Optional[str]
+ A string containing the dimension ordering desired for the returned ndarray.
+ Default: The current image dimensions. i.e. `self.dims`
kwargs:
C=1: specifies Channel 1
@@ -270,23 +308,147 @@ def get_image_data(
Returns
-------
- ndarray with dimension ordering that was specified with out_orientation
- Note: if a requested dimension is not present in the data the dimension is added with
- a depth of 1. The default return dimensions are "STCZYX".
+ data: dask array
+ The read data with the dimension ordering that was specified with out_orientation.
+
+ Note: If a requested dimension is not present in the data the dimension is added with
+ a depth of 1.
"""
- out_orientation = self.dims if out_orientation is None else out_orientation
- if out_orientation == self.dims:
- return self.data
+ # If no out orientation, simply return current data as numpy array
+ if out_orientation is None:
+ return self.dask_data
+
+ # Transform and return
return transforms.reshape_data(
- data=self.data,
+ data=self.dask_data,
given_dims=self.dims,
return_dims=out_orientation,
- copy=copy,
**kwargs,
)
- def get_channel_names(self, scene: int = 0):
+ def get_image_data(self, out_orientation: Optional[str] = None, **kwargs) -> np.ndarray:
"""
+ Get specific dimension image data out of an image as a numpy array.
+
+ Parameters
+ ----------
+ out_orientation: Optional[str]
+ A string containing the dimension ordering desired for the returned ndarray.
+ Default: The current image dimensions. i.e. `self.dims`
+
+ kwargs:
+ C=1: specifies Channel 1
+ T=3: specifies the fourth index in T
+ D=n: D is Dimension letter and n is the index desired D should not be present in the out_orientation
+
+ Returns
+ -------
+ data: np.ndarray
+ The read data with the dimension ordering that was specified with out_orientation.
+
+ Note: If a requested dimension is not present in the data the dimension is added with
+ a depth of 1.
+ """
+ return self.get_image_dask_data(out_orientation=out_orientation, **kwargs).compute()
+
+ def view_napari(self, rgb: bool = False, **kwargs):
+ """
+ If installed, load the image in a napari viewer.
+
+ Parameters
+ ----------
+ rgb: bool
+ Is the image RGB / RGBA
+ Default: False (is not RGB)
+ **kwargs
+ Extra arguments passed down to the viewer
+ """
+ try:
+ import napari
+
+ # Construct getitem operations tuple to select down the data in the filled dimensions
+ ops = []
+ selected_dims = []
+ for dim in self.dims:
+ if self.size(dim)[0] == 1:
+ ops.append(0)
+ else:
+ ops.append(slice(None, None, None))
+ selected_dims.append(dim)
+
+ # Actually select the down
+ data = self.dask_data[tuple(ops)]
+
+ # Convert selected_dims to string
+ dims = "".join(selected_dims)
+
+ # Create name for window
+ if isinstance(self.reader, ArrayLikeReader):
+ title = f"napari: {self.dask_data.shape}"
+ else:
+ title = f"napari: {self.reader._file.name}"
+
+ # Handle RGB entirely differently
+ if rgb:
+ # Swap channel to last dimension
+ new_dims = f"{dims.replace(Dimensions.Channel, '')}{Dimensions.Channel}"
+ data = transforms.transpose_to_dims(
+ data=data,
+ given_dims=dims,
+ return_dims=new_dims
+ )
+
+ # Run napari
+ with napari.gui_qt():
+ napari.view_image(
+ data,
+ is_pyramid=False,
+ ndisplay=3 if Dimensions.SpatialZ in dims else 2,
+ title=title,
+ axis_labels=dims.replace(Dimensions.Channel, ""),
+ rgb=rgb,
+ **kwargs
+ )
+
+ # Handle all other images besides RGB not requested
+ else:
+ # Channel axis
+ c_axis = dims.index(Dimensions.Channel) if Dimensions.Channel in dims else None
+
+ # Set visible based on number of channels
+ if c_axis is not None:
+ if data.shape[c_axis] > 3:
+ visible = False
+ else:
+ visible = True
+ else:
+ visible = True
+
+ # Drop channel from dims string
+ dims = dims.replace(Dimensions.Channel, "") if Dimensions.Channel in dims else dims
+
+ # Run napari
+ with napari.gui_qt():
+ napari.view_image(
+ data,
+ is_pyramid=False,
+ ndisplay=3 if Dimensions.SpatialZ in dims else 2,
+ channel_axis=c_axis,
+ axis_labels=dims,
+ title=title,
+ visible=visible,
+ **kwargs
+ )
+
+ except ModuleNotFoundError:
+ raise ModuleNotFoundError(
+ f"'napari' has not been installed. To use this function install napari with either: "
+ f"'pip install napari' or 'pip install aicsimageio[interactive]'"
+ )
+
+ def get_channel_names(self, scene: int = 0) -> List[str]:
+ """
+ Get the image's channel names.
Parameters
----------
@@ -297,25 +459,80 @@ def get_channel_names(self, scene: int = 0):
list of strings representing the channel names
"""
try:
- names = self._reader.get_channel_names(scene)
+ names = self.reader.get_channel_names(scene)
except AttributeError:
names = [str(i) for i in range(self.size_c)]
return names
- def close(self):
- self.reader.close()
-
def __repr__(self) -> str:
return f""
+ @property
+ def cluster(self) -> Optional[LocalCluster]:
+ return self._cluster
+
+ @property
+ def client(self) -> Optional[Client]:
+ return self._client
+
+ def close(self):
+ """
+ Always close the Dask Client connection.
+ If connected to *strictly* a LocalCluster, close it down as well.
+ """
+ self._cluster, self._client = dask_utils.shutdown_cluster_and_client(self.cluster, self.client)
+
def __enter__(self):
+ """
+ If provided an address, create a Dask Client connection.
+ If not provided an address, create a LocalCluster and Client connection.
+ If not provided an address, other Dask kwargs are accepted and passed down to the LocalCluster object.
+ """
+ self._cluster, self._client = dask_utils.spawn_cluster_and_client(**self._dask_kwargs)
+
return self
def __exit__(self, exc_type, exc_val, exc_tb):
+ """
+ Always close the Dask Client connection.
+ If connected to *strictly* a LocalCluster, close it down as well.
+ """
self.close()
+def imread_dask(data: types.ImageLike, **kwargs) -> da.core.Array:
+ """
+ Read image as a dask array.
+
+ Parameters
+ ----------
+ data: types.ImageLike
+ A filepath, in memory numpy array, or preconfigured dask array.
+ kwargs: Dict[str, Any]
+ Any extra arguments to passed down to AICSImage and subsequent readers.
+
+ Returns
+ -------
+ data: da.core.Array
+ The image read and configured as a dask array.
+ """
+ return AICSImage(data, **kwargs).dask_data
+
+
def imread(data: types.ImageLike, **kwargs) -> np.ndarray:
- with AICSImage(data, **kwargs) as image:
- imagedata = image.data
- return imagedata
+ """
+ Read image as a numpy ndarray.
+
+ Parameters
+ ----------
+ data: types.ImageLike
+ A filepath, in memory numpy array, or preconfigured dask array.
+ kwargs: Dict[str, Any]
+ Any extra arguments to passed down to AICSImage and subsequent readers.
+
+ Returns
+ -------
+ data: np.ndarray
+ The image read and configured as a numpy ndarray.
+ """
+ return AICSImage(data, **kwargs).data
diff --git a/aicsimageio/constants.py b/aicsimageio/constants.py
index f9b771cd7..4d694d62e 100644
--- a/aicsimageio/constants.py
+++ b/aicsimageio/constants.py
@@ -1,4 +1,13 @@
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
-DEFAULT_DIMENSION_ORDER = "STCZYX"
+class Dimensions:
+ Scene = "S"
+ Time = "T"
+ Channel = "C"
+ SpatialZ = "Z"
+ SpatialY = "Y"
+ SpatialX = "X"
+ DefaultOrderList = [Scene, Time, Channel, SpatialZ, SpatialY, SpatialX]
+ DefaultOrder = "".join(DefaultOrderList)
diff --git a/aicsimageio/dask_utils.py b/aicsimageio/dask_utils.py
new file mode 100644
index 000000000..909e7c7ab
--- /dev/null
+++ b/aicsimageio/dask_utils.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+from contextlib import contextmanager
+from typing import Optional, Tuple
+
+from distributed import Client, LocalCluster
+
+###############################################################################
+
+log = logging.getLogger(__name__)
+
+###############################################################################
+
+
+def spawn_cluster_and_client(
+ address: Optional[str] = None,
+ **kwargs
+) -> Tuple[Optional[LocalCluster], Optional[Client]]:
+ """
+ If provided an address, create a Dask Client connection.
+ If not provided an address, create a LocalCluster and Client connection.
+ If not provided an address, other Dask kwargs are accepted and passed down to the LocalCluster object.
+ """
+ if address is not None:
+ client = Client(address)
+ log.info(f"Connected to Remote Dask Cluster: {client}")
+ else:
+ cluster = LocalCluster(**kwargs)
+ client = Client(cluster)
+ log.info(f"Connected to Local Dask Cluster: {client}")
+
+ return cluster, client
+
+
+def shutdown_cluster_and_client(
+ cluster: Optional[LocalCluster],
+ client: Optional[Client]
+) -> Tuple[Optional[LocalCluster], Optional[Client]]:
+ """
+ Shutdown a cluster and client.
+ """
+ if cluster is not None:
+ cluster.close()
+ if client is not None:
+ client.shutdown()
+ client.close()
+
+ return cluster, client
+
+
+@contextmanager
+def cluster_and_client(address: Optional[str] = None, **kwargs):
+ """
+ If provided an address, create a Dask Client connection.
+ If not provided an address, create a LocalCluster and Client connection.
+ If not provided an address, other Dask kwargs are accepted and passed down to the LocalCluster object.
+
+ These objects will only live for the duration of this context manager.
+
+ Example
+ -------
+ ```
+ with cluster_and_client() as (cluster, client):
+ img1 = AICSImage("1.tiff")
+ img2 = AICSImage("2.czi")
+
+ # other processing
+ ```
+ """
+ try:
+ cluster, client = spawn_cluster_and_client(address=address, **kwargs)
+ yield cluster, client
+ finally:
+ shutdown_cluster_and_client(cluster=cluster, client=client)
diff --git a/aicsimageio/exceptions.py b/aicsimageio/exceptions.py
index 379af1fd3..375cac1c0 100644
--- a/aicsimageio/exceptions.py
+++ b/aicsimageio/exceptions.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
class UnsupportedFileFormatError(Exception):
@@ -6,12 +8,12 @@ class UnsupportedFileFormatError(Exception):
the supported file types and cannot be parsed with AICSImage.
"""
- def __init__(self, type_, **kwargs):
+ def __init__(self, data, **kwargs):
super().__init__(**kwargs)
- self.type_ = type_
+ self.data = data
def __str__(self):
- return f"AICSImage module does not support this image file type: '{self.type_}'"
+ return f"AICSImage module does not support this image file type: '{self.data}'."
class InvalidDimensionOrderingError(Exception):
@@ -28,17 +30,15 @@ def __str__(self):
return self.message
-class MultiSceneCziException(Exception):
+class ConflictingArgumentsError(Exception):
"""
- This exception is intended to be thrown when a CZI file has multiple scenes. This is only to
- be thrown if the Reader is given a multi-scene CZI files and the backend library isn't able
- to read multi-scene CZI.
+ This exception is returned when 2 arguments to the same function are in conflict.
"""
pass
-class ConflictingArgumentsError(Exception):
+class InconsistentShapeError(Exception):
"""
- This exception is returned when 2 arguments to the same function are in conflict.
+ A general function to use when the shape returned or requested from an array operation is invalid.
"""
pass
diff --git a/aicsimageio/readers/__init__.py b/aicsimageio/readers/__init__.py
index 0ad083cda..32b309068 100644
--- a/aicsimageio/readers/__init__.py
+++ b/aicsimageio/readers/__init__.py
@@ -1,5 +1,8 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from .arraylike_reader import ArrayLikeReader # noqa: F401
from .czi_reader import CziReader # noqa: F401
from .default_reader import DefaultReader # noqa: F401
-from .ndarray_reader import NdArrayReader # noqa: F401
from .ome_tiff_reader import OmeTiffReader # noqa: F401
from .tiff_reader import TiffReader # noqa: F401
diff --git a/aicsimageio/readers/ndarray_reader.py b/aicsimageio/readers/arraylike_reader.py
similarity index 50%
rename from aicsimageio/readers/ndarray_reader.py
rename to aicsimageio/readers/arraylike_reader.py
index 1c6277bea..4c3d97f6a 100644
--- a/aicsimageio/readers/ndarray_reader.py
+++ b/aicsimageio/readers/arraylike_reader.py
@@ -1,22 +1,23 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+import dask.array as da
import numpy as np
-from .. import exceptions
+from .. import exceptions, types
from .reader import Reader
+###############################################################################
-class NdArrayReader(Reader):
- """
- NdArrayReader(np.ones((1,2,3)))
- A catch all for numpy ndarray reading.
+class ArrayLikeReader(Reader):
+ """
+ A catch all for numpy ndarray and dask array reading.
Parameters
----------
- arr: numpy.ndarray
- An in memory numpy ndarray.
+ data: Union[numpy.ndarray, da.core.Array]
+ An in memory numpy ndarray or preconfigured dask array.
Notes
-----
@@ -24,13 +25,21 @@ class NdArrayReader(Reader):
returned with dimensions assumed in order but with extra dimensions removed depending on image shape.
"""
- def __init__(self, arr: np.ndarray):
- self._data = arr
- self._dims = self.guess_dim_order(self.data.shape)
+ def __init__(self, data: types.ArrayLike, **kwargs):
+ # Store data as dask array
+ if isinstance(data, np.ndarray):
+ self._dask_data = da.from_array(data)
+ elif isinstance(data, da.core.Array):
+ self._dask_data = data
+ else:
+ raise TypeError(data)
+
+ # Guess dims
+ self._dims = self.guess_dim_order(self.dask_data.shape)
@property
- def data(self) -> np.ndarray:
- return self._data
+ def dask_data(self) -> da.core.Array:
+ return self._dask_data
@property
def dims(self) -> str:
@@ -39,9 +48,9 @@ def dims(self) -> str:
@dims.setter
def dims(self, dims: str):
# Check amount of provided dims against data shape
- if len(dims) != len(self.data.shape):
+ if len(dims) != len(self.dask_data.shape):
raise exceptions.InvalidDimensionOrderingError(
- f"Provided too many dimensions for the associated array. "
+ f"Provided too many dimensions for the associated file. "
f"Received {len(dims)} dimensions [dims: {dims}] "
f"for image with {len(self.data.shape)} dimensions [shape: {self.data.shape}]."
)
@@ -54,8 +63,5 @@ def metadata(self) -> None:
return None
@staticmethod
- def _is_this_type(arr: np.ndarray) -> bool:
- return isinstance(arr, np.ndarray)
-
- def close(self) -> None:
- pass
+ def _is_this_type(arr: types.ArrayLike) -> bool:
+ return isinstance(arr, (np.ndarray, da.core.Array))
diff --git a/aicsimageio/readers/czi_reader.py b/aicsimageio/readers/czi_reader.py
index c79c61251..028e7056c 100644
--- a/aicsimageio/readers/czi_reader.py
+++ b/aicsimageio/readers/czi_reader.py
@@ -1,218 +1,402 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import io
import logging
-import warnings
-from typing import Optional, Tuple
-from xml.etree import ElementTree
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+import dask.array as da
import numpy as np
+from aicspylibczi import CziFile
+from dask import delayed
+from lxml.etree import _Element
-from aicsimageio import types
-
+from .. import exceptions, types
from ..buffer_reader import BufferReader
-from ..exceptions import MultiSceneCziException, UnsupportedFileFormatError
+from ..constants import Dimensions
from .reader import Reader
-with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- from ..vendor import czifile
+###############################################################################
log = logging.getLogger(__name__)
+###############################################################################
+
class CziReader(Reader):
"""
- CziReader is intended for reading single scene Czi files. It is meant to handle the specifics of using the backend
- library to create a unified interface. This enables higher level functions to duck type the File Readers.
+ CziReader wraps aicspylibczi to provide the same reading capabilities but abstracts the specifics of using the
+ backend library to create a unified interface. This enables higher level functions to duck type the File Readers.
+
+ Parameters
+ ----------
+ data: types.FileLike
+ A string or path to the CZI file to be read.
+ chunk_by_dims: List[str]
+ The dimensions to use as the for mapping the chunks / blocks.
+ Default: [Dimensions.SpatialZ, Dimensions.SpatialY, Dimensions.SpatialX]
+ Note: SpatialY and SpatialX will always be added to the list if not present.
+ S: int
+ If the image has different dimensions on any scene from another, the dask array construction will fail.
+ In that case, use this parameter to specify a specific scene to construct a dask array for.
+ Default: 0 (select the first scene)
"""
ZEISS_2BYTE = b'ZI' # First two characters of a czi file according to Zeiss docs
ZEISS_10BYTE = b'ZISRAWFILE' # First 10 characters of a well formatted czi file.
- def __init__(self, file: types.FileLike, max_workers: Optional[int] = None, **kwargs):
- """
-
- Parameters
- ----------
- file : a file like object ("Filename.czi", Path("/path/Filename.czi") or an open stream to the data
- max_workers : (Optional) the number of cores the backend library is allowed to use to load the data in the file.
- """
- super().__init__(file, **kwargs)
- try:
- self.czi = czifile.CziFile(self._bytes)
- except Exception:
- log.error("czifile could not parse this input")
- raise UnsupportedFileFormatError("exception from with CziFile backend library.")
-
- if self._is_multiscene():
- raise MultiSceneCziException(
- "File is Multiscene. The backend library CziFile can only read single scene images."
- )
+ def __init__(
+ self,
+ data: types.FileLike,
+ chunk_by_dims: List[str] = [Dimensions.SpatialZ, Dimensions.SpatialY, Dimensions.SpatialX],
+ S: int = 0,
+ **kwargs
+ ):
+ # Run super init to check filepath provided
+ super().__init__(data, **kwargs)
- self._max_workers = max_workers
+ # Store parameters needed for _daread
+ self.chunk_by_dims = chunk_by_dims
+ self.specific_s_index = S
@staticmethod
- def _is_this_type(buffer: io.BufferedIOBase) -> bool:
+ def _is_this_type(buffer: io.BytesIO) -> bool:
with BufferReader(buffer) as buffer_reader:
if buffer_reader.endianness != CziReader.ZEISS_2BYTE:
return False
header = buffer_reader.endianness + buffer_reader.read_bytes(8)
return header == CziReader.ZEISS_10BYTE
- @property
- def data(self) -> np.ndarray:
- """
- Returns
- -------
- the data from the czi file with the native order (i.e. "TZCYX")
+ @staticmethod
+ def _read_image(img: Path, read_dims: Optional[Dict[str, int]] = None) -> Tuple[np.ndarray, List[Tuple[str, int]]]:
"""
- if self._data is None:
- # load the data
- self._data = self.czi.asarray(max_workers=self._max_workers)
- return self._data
+ Read and return the squeezed image data requested along with the dimension info that was read.
- @property
- def dims(self) -> str:
- """
- Returns
- -------
- The native shape of the image.
- """
- return self.czi.axes
+ Parameters
+ ----------
+ img: Path
+ Path to the CZI file to read.
+ read_dims: Optional[Dict[str, int]]
+ The dimensions to read from the file as a dictionary of string to integer.
+ Default: None (Read all data from the image)
- @property
- def metadata(self) -> ElementTree:
- """
- Lazy load the metadata from the CZI file
Returns
-------
- The xml Element Tree of the metadata
+ data: np.ndarray
+ The data read for the dimensions provided.
+ read_dimensions: List[Tuple[str, int]]]
+ The dimension sizes that were returned from the read.
"""
- if self._metadata is None:
- # load the metadata
- self._metadata = self.czi.metadata
- return self._metadata
+ # Catch optional read dim
+ if read_dims is None:
+ read_dims = {}
+
+ # Init czi
+ czi = CziFile(img)
+
+ # Read image
+ log.debug(f"Reading dimensions: {read_dims}")
+ data, dims = czi.read_image(**read_dims)
+
+ # Drop dims that shouldn't be provided back
+ ops = []
+ real_dims = []
+ for i, dim_info in enumerate(dims):
+ # Expand dimension info
+ dim, size = dim_info
+
+ # If the dim was provided in the read dims we know a single plane for that
+ # dimension was requested so remove it
+ if dim in read_dims:
+ ops.append(0)
+ # Otherwise just read the full slice
+ else:
+ ops.append(slice(None, None, None))
+ real_dims.append(dim_info)
+
+ # Convert ops and run getitem
+ return data[tuple(ops)], real_dims
- def close(self):
- """
- Close the czi file handle and perform any upstream cleanup
- Returns
- -------
- None
- """
- self.czi.close()
- super().close()
+ @staticmethod
+ def _imread(img: Path, read_dims: Optional[Dict[str, str]] = None) -> np.ndarray:
+ data, dims = CziReader._read_image(img=img, read_dims=read_dims)
+ return data
- def dtype(self) -> np.dtype:
- """
- Returns
- -------
- the data type of the ndarray being returned (uint16, uint8, etc)
+ @staticmethod
+ def _daread(
+ img: Path,
+ czi: CziFile,
+ chunk_by_dims: List[str] = [Dimensions.SpatialZ, Dimensions.SpatialY, Dimensions.SpatialX],
+ S: int = 0
+ ) -> Tuple[da.core.Array, str]:
"""
- return self.czi.dtype
+ Read a CZI image file as a delayed dask array where certain dimensions act as the chunk size.
- def size_s(self):
- """
- Returns
- -------
- The number of scenes in the data
- """
- return self._size_of_dimension('S')
+ Parameters
+ ----------
+ img: Path
+ The filepath to read.
+ czi: CziFile
+ The loaded CziFile object created from reading the filepath.
+ chunk_by_dims: List[str]
+ The dimensions to use as the for mapping the chunks / blocks.
+ Default: [Dimensions.SpatialZ, Dimensions.SpatialY, Dimensions.SpatialX]
+ Note: SpatialY and SpatialX will always be added to the list if not present.
+ S: int
+ If the image has different dimensions on any scene from another, the dask array construction will fail.
+ In that case, use this parameter to specify a specific scene to construct a dask array for.
+ Default: 0 (select the first scene)
- def size_z(self):
- """
Returns
-------
- The number of Z slices in the stack
+ img: dask.array.core.Array
+ The constructed dask array where certain dimensions are chunked.
+ dims: str
+ The dimension order as a string.
"""
- return self._size_of_dimension('Z')
+ # Get image dims indicies
+ image_dim_indices = czi.dims_shape()
+
+ # Catch inconsistent scene dimension sizes
+ if len(image_dim_indices) > 1:
+ # Choose the provided scene
+ try:
+ image_dim_indices = image_dim_indices[S]
+ log.info(f"File contains variable dimensions per scene, selected scene: {S} for data retrieval.")
+ except IndexError:
+ raise exceptions.InconsistentShapeError(
+ f"The CZI image provided has variable dimensions per scene. "
+ f"Please provide a valid index to the 'S' parameter to create a dask array for the index provided. "
+ f"Provided scene index: {S}. Scene index range: 0-{len(image_dim_indices)}."
+ )
+ else:
+ # If the list is length one that means that all the scenes in the image have the same dimensions
+ # Just select the first dictionary in the list
+ image_dim_indices = image_dim_indices[0]
+
+ # Uppercase dimensions provided to chunk by dims
+ chunk_by_dims = [d.upper() for d in chunk_by_dims]
+
+ # Always add Y and X dims to chunk by dims because that is how CZI files work
+ if Dimensions.SpatialY not in chunk_by_dims:
+ log.info(f"Adding the Spatial Y dimension to chunk by dimensions as it was not found.")
+ chunk_by_dims.append(Dimensions.SpatialY)
+ if Dimensions.SpatialX not in chunk_by_dims:
+ log.info(f"Adding the Spatial X dimension to chunk by dimensions as it was not found.")
+ chunk_by_dims.append(Dimensions.SpatialX)
+
+ # Setup read dimensions for an example chunk
+ first_chunk_read_dims = {}
+ for dim, (dim_begin_index, dim_end_index) in image_dim_indices.items():
+ # Only add the dimension if the dimension isn't a part of the chunk
+ if dim not in chunk_by_dims:
+ # Add to read dims
+ first_chunk_read_dims[dim] = dim_begin_index
+
+ # Read first chunk for information used by dask.array.from_delayed
+ sample, sample_dims = czi.read_image(**first_chunk_read_dims)
+
+ # Get the shape for the chunk and operating shape for the dask array
+ # We also collect the chunk and non chunk dimension ordering so that we can swap the dimensions after we
+ # block the dask array together.
+ sample_chunk_shape = []
+ operating_shape = []
+ non_chunk_dimension_ordering = []
+ chunk_dimension_ordering = []
+ for i, dim_info in enumerate(sample_dims):
+ # Unpack dim info
+ dim, size = dim_info
+
+ # If the dim is part of the specified chunk dims then append it to the sample, and, append the dimension
+ # to the chunk dimension ordering
+ if dim in chunk_by_dims:
+ sample_chunk_shape.append(size)
+ chunk_dimension_ordering.append(dim)
+
+ # Otherwise, append the dimension to the non chunk dimension ordering, and, append the true size of the
+ # image at that dimension
+ else:
+ non_chunk_dimension_ordering.append(dim)
+ operating_shape.append(image_dim_indices[dim][1] - image_dim_indices[dim][0])
+
+ # Convert shapes to tuples and combine the non and chunked dimension orders as that is the order the data will
+ # actually come out of the read data as
+ sample_chunk_shape = tuple(sample_chunk_shape)
+ blocked_dimension_order = non_chunk_dimension_ordering + chunk_dimension_ordering
+
+ # Fill out the rest of the operating shape with dimension sizes of 1 to match the length of the sample chunk
+ # When dask.block happens it fills the dimensions from inner-most to outer-most with the chunks as long as
+ # the dimension is size 1
+ # Basically, we are adding empty dimensions to the operating shape that will be filled by the chunks from dask
+ operating_shape = tuple(operating_shape) + (1, ) * len(sample_chunk_shape)
+
+ # Create empty numpy array with the operating shape so that we can iter through and use the multi_index to
+ # create the readers.
+ lazy_arrays = np.ndarray(operating_shape, dtype=object)
+
+ # We can enumerate over the multi-indexed array and construct read_dims dictionaries by simply zipping together
+ # the ordered dims list and the current multi-index plus the begin index for that plane. We then set the value
+ # of the array at the same multi-index to the delayed reader using the constructed read_dims dictionary.
+ dims = [d for d in czi.dims]
+ begin_indicies = tuple(image_dim_indices[d][0] for d in dims)
+ for i, _ in np.ndenumerate(lazy_arrays):
+ # Add the czi file begin index for each dimension to the array dimension index
+ this_chunk_read_indicies = (
+ current_dim_begin_index + curr_dim_index
+ for current_dim_begin_index, curr_dim_index in zip(begin_indicies, i)
+ )
- def size_c(self):
- """
- Returns
- -------
- The number of Channels present in the data
- """
- return self._size_of_dimension('C')
+ # Zip the dims with the read indices
+ this_chunk_read_dims = dict(zip(blocked_dimension_order, this_chunk_read_indicies))
- def size_t(self):
- """
- Returns
- -------
- The number of time steps in the data
- """
- return self._size_of_dimension('T')
+ # Remove the dimensions that we want to chunk by from the read dims
+ for d in chunk_by_dims:
+ if d in this_chunk_read_dims:
+ this_chunk_read_dims.pop(d)
- def size_x(self):
- """
- Returns
- -------
- The number of pixels in the images X axis
- """
- return self._size_of_dimension('X')
+ # Add delayed array to lazy arrays at index
+ lazy_arrays[i] = da.from_delayed(
+ delayed(CziReader._imread)(img, this_chunk_read_dims),
+ shape=sample_chunk_shape,
+ dtype=sample.dtype,
+ )
- def size_y(self):
- """
- Returns
- -------
- The number of pixels in the images Y axis
- """
- return self._size_of_dimension('Y')
+ # Convert the numpy array of lazy readers into a dask array and fill the inner-most empty dimensions with chunks
+ merged = da.block(lazy_arrays.tolist())
+
+ # Because we have set certain dimensions to be chunked and others not
+ # we will need to transpose back to original dimension ordering
+ # Example being, if the original dimension ordering was "SZYX" and we want to chunk by "S", "Y", and "X"
+ # We created an array with dimensions ordering "ZSYX"
+ transpose_indices = []
+ transpose_required = False
+ for i, d in enumerate(czi.dims):
+ new_index = blocked_dimension_order.index(d)
+ if new_index != i:
+ transpose_required = True
+ transpose_indices.append(new_index)
+ else:
+ transpose_indices.append(i)
+
+ # Only run if the transpose is actually required
+ # The default case is "Z", "Y", "X", which _usually_ doesn't need to be transposed because that is _usually_
+ # The normal dimension order of the CZI file anyway
+ if transpose_required:
+ merged = da.transpose(merged, tuple(transpose_indices))
+
+ # Because dimensions outside of Y and X can be in any order and present or not
+ # we also return the dimension order string.
+ return merged, "".join(dims)
- def _size_of_dimension(self, dimension: str) -> int:
+ @staticmethod
+ def _daread_safe(
+ img: Union[str, Path],
+ chunk_by_dims: List[str] = [Dimensions.SpatialZ, Dimensions.SpatialY, Dimensions.SpatialX],
+ S: int = 0
+ ) -> Tuple[da.core.Array, str]:
"""
+ Safely read a CZI image file as a delayed dask array where certain dimensions act as the chunk size.
+
Parameters
----------
- dimension : str (a single character)
-
- Raises
- ------
- If a string of length greater or smaller than 1 is passed in raise a TypeError
+ img: Union[str, Path]
+ The filepath to read.
+ chunk_by_dims: List[str]
+ The dimensions to use as the for mapping the chunks / blocks.
+ Default: [Dimensions.SpatialZ, Dimensions.SpatialY, Dimensions.SpatialX]
+ Note: SpatialY and SpatialX will always be added to the list if not present.
+ S: int
+ If the image has different dimensions on any scene from another, the dask array construction will fail.
+ In that case, use this parameter to specify a specific scene to construct a dask array for.
+ Default: 0 (select the first scene)
Returns
-------
- The size of the dimension in the data, if the dimension is not found in the "BTCZYX" type string
- then the default dimension size of 1 is returned.
+ img: dask.array.core.Array
+ The constructed dask array where certain dimensions are chunked.
+ dims: str
+ The dimension order as a string.
"""
+ # Resolve image path
+ img = CziReader._resolve_image_path(img)
- index = self._lookup_dimension_index(dimension)
- if index == -1:
- return 1
- return self.czi.shape[index]
+ # Init temp czi
+ czi = CziFile(img)
- def _lookup_dimension_index(self, dimension: str) -> int:
- """
- Use the axes metadata in the czi file to find the dimension index, additionally this
- function should be used for determining if a Dimension is present in the native data.
- Parameters
- ----------
- dimension : str (a single character)
- sensible values are any one of ('V', 'H', 'M', 'B', 'I', 'S', 'T', 'R', 'Z', 'C', 'Y', 'X')
- most likely values are one of ('S', 'T', 'C', 'Z', 'Y', 'X')
+ # Safely construct the dask array or catch any exception
+ try:
+ return CziReader._daread(img=img, czi=czi, chunk_by_dims=chunk_by_dims, S=S)
- Raises
- ------
- If a string of length greater or smaller than 1 is passed in raise a TypeError
+ except Exception as e:
+ # A really bad way to close any connection to the CZI object
+ czi._bytes = None
+ czi.reader = None
+ raise e
+
+ @property
+ def dask_data(self) -> da.core.Array:
+ """
Returns
-------
- the integer position of the channel or -1 if the character is not present in the file description
+ Constructed dask array where each chunk is a delayed read from the CZI file.
+ Places dimensions in the native order (i.e. "TZCYX")
"""
- if len(dimension) != 1:
- raise TypeError(f"channel lookup requested with channel {dimension}")
- return self.czi.axes.find(dimension)
+ if self._dask_data is None:
+ self._dask_data, self._dims = CziReader._daread_safe(
+ self._file,
+ chunk_by_dims=self.chunk_by_dims,
+ S=self.specific_s_index
+ )
+
+ return self._dask_data
+
+ @property
+ def dims(self) -> str:
+ if self._dims is None:
+ self._dims = CziFile(self._file).dims
+ return self._dims
+
+ def dtype(self) -> np.dtype:
+ return self.dask_data.dtype
- def _is_multiscene(self):
+ @property
+ def metadata(self) -> _Element:
"""
- Check if the metadata the czi is multiscene
+ Lazy load the metadata from the CZI file
Returns
-------
- True if multi-scene, False if single-scene
+ The xml Element Tree of the metadata
"""
- index = self._lookup_dimension_index('S')
- if index < 0:
- return False
- img_shape = self.czi.filtered_subblock_directory[0].shape
- return img_shape[index] != 1
+ if self._metadata is None:
+ # load the metadata
+ self._metadata = CziFile(self._file).meta
+ return self._metadata
+
+ def _size_of_dimension(self, dim: str) -> int:
+ if dim in self.dims:
+ return self.dask_data.shape[self.dims.index(dim)]
+
+ return 1
+
+ def size_s(self) -> int:
+ return self._size_of_dimension(Dimensions.Scene)
+
+ def size_t(self) -> int:
+ return self._size_of_dimension(Dimensions.Time)
+
+ def size_c(self) -> int:
+ return self._size_of_dimension(Dimensions.Channel)
+
+ def size_z(self) -> int:
+ return self._size_of_dimension(Dimensions.SpatialZ)
+
+ def size_y(self) -> int:
+ return self._size_of_dimension(Dimensions.SpatialY)
+
+ def size_x(self) -> int:
+ return self._size_of_dimension(Dimensions.SpatialX)
def get_channel_names(self, scene: int = 0):
chelem = self.metadata.findall("./Metadata/Information/Image/Dimensions/Channels/Channel")
diff --git a/aicsimageio/readers/default_reader.py b/aicsimageio/readers/default_reader.py
index 2f37fbb30..531710835 100644
--- a/aicsimageio/readers/default_reader.py
+++ b/aicsimageio/readers/default_reader.py
@@ -2,69 +2,98 @@
# -*- coding: utf-8 -*-
import io
+from pathlib import Path
+from typing import Any, Dict
+import dask.array as da
import imageio
import numpy as np
+from dask import delayed
from .. import exceptions
from .reader import Reader
+###############################################################################
+
class DefaultReader(Reader):
"""
- DefaultReader('file.jpg')
-
- A catch all for image file reading that uses imageio as its back end.
+ A catch all for image file reading that uses imageio for reading.
Parameters
----------
- file: types.FileLike
- A path or bytes object to read from.
-
- Notes
- -----
- Because this is simply a wrapper around imageio, no metadata is returned. However, dimension order is returned with
- dimensions assumed in order but with extra dimensions removed depending on image shape.
-
- This reader always returns the first YX plane of an image. If you need to read GIFs for example, please use
- `imageio.mimread` to get a list of YXC planes for the frames of the GIF.
-
- For more details about how `imageio.imread`, differs from `imageio.volread` and `imageio.mimread`, please refer to
- their documentation.
+ file: types.ImageLike
+ String with path to file.
"""
- @property
- def data(self) -> np.ndarray:
- if self._data is None:
- self._data = imageio.imread(self._bytes)
+ @staticmethod
+ def _get_data(file: Path, index: int) -> np.ndarray:
+ with imageio.get_reader(file) as reader:
+ return np.asarray(reader.get_data(index))
- return self._data
+ @property
+ def dask_data(self) -> da.core.Array:
+ # Construct delayed many image reads
+ if self._dask_data is None:
+ try:
+ with imageio.get_reader(self._file) as reader:
+ # Store length as it is used a bunch
+ image_length = reader.get_length()
+
+ # Handle single image formats like png, jpeg, etc
+ if image_length == 1:
+ self._dask_data = da.from_array(self._get_data(self._file, 0))
+
+ # Handle many image formats like gif, mp4, etc
+ elif image_length > 1:
+ # Get a sample image
+ sample = self._get_data(self._file, 0)
+
+ # Create operating shape for the final dask array by prepending image length to a tuple of
+ # ones that is the same length as the sample shape
+ operating_shape = (image_length, ) + ((1, ) * len(sample.shape))
+ # Create numpy array of empty arrays for delayed get data functions
+ lazy_arrays = np.ndarray(operating_shape, dtype=object)
+ for indicies, _ in np.ndenumerate(lazy_arrays):
+ lazy_arrays[indicies] = da.from_delayed(
+ delayed(self._get_data)(self._file, indicies[0]),
+ shape=sample.shape,
+ dtype=sample.dtype
+ )
+
+ # Block them into a single dask array
+ self._dask_data = da.block(lazy_arrays.tolist())
+
+ # Catch all other image types as unsupported
+ # https://imageio.readthedocs.io/en/stable/userapi.html#imageio.core.format.Reader.get_length
+ else:
+ exceptions.UnsupportedFileFormatError(self._file)
+
+ # Reraise unsupported file format
+ except exceptions.UnsupportedFileFormatError:
+ raise exceptions.UnsupportedFileFormatError(self._file)
+
+ return self._dask_data
@property
def dims(self) -> str:
- """
- `imageio.imread` returns the first YX plane of an image, except in the case where the image is RGB / RGBA, in
- which case it returns the first YXC plane of an image.
-
- This dims property handles those cases by making assumptions about the dimension order based off the shape of
- the data stored in the reader. In the case where more data than an YXC plane is returned, it uses the
- `guess_dim_order` function. However, I can't tell when that should ever get run and is really just used as a
- safety catch all.
- """
+ # Set dims if not set
if self._dims is None:
- if len(self.data.shape) == 2:
+ if len(self.dask_data.shape) == 2:
self._dims = "YX"
- elif len(self.data.shape) == 3:
+ elif len(self.dask_data.shape) == 3:
self._dims = "YXC"
+ elif len(self.dask_data.shape) == 4:
+ self._dims = "TYXC"
else:
- self._dims = self.guess_dim_order(self.data.shape)
+ self._dims = self.guess_dim_order(self.dask_data.shape)
return self._dims
@dims.setter
def dims(self, dims: str):
# Check amount of provided dims against data shape
- if len(dims) != len(self.data.shape):
+ if len(dims) != len(self.dask_data.shape):
raise exceptions.InvalidDimensionOrderingError(
f"Provided too many dimensions for the associated file. "
f"Received {len(dims)} dimensions [dims: {dims}] "
@@ -75,13 +104,18 @@ def dims(self, dims: str):
self._dims = dims
@property
- def metadata(self) -> None:
- return None
+ def metadata(self) -> Dict[str, Any]:
+ if self._metadata is None:
+ with imageio.get_reader(self._file) as reader:
+ self._metadata = reader.get_meta_data()
+
+ return self._metadata
@staticmethod
- def _is_this_type(buffer: io.BufferedIOBase) -> bool:
+ def _is_this_type(buffer: io.BytesIO) -> bool:
+ # Use imageio to check if they have a reader for this file
try:
- imageio.get_reader(buffer)
- return True
+ with imageio.get_reader(buffer):
+ return True
except ValueError:
return False
diff --git a/aicsimageio/readers/ome_tiff_reader.py b/aicsimageio/readers/ome_tiff_reader.py
index e74644209..bc46cb1c9 100644
--- a/aicsimageio/readers/ome_tiff_reader.py
+++ b/aicsimageio/readers/ome_tiff_reader.py
@@ -1,41 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import io
import logging
import re
-
-import numpy as np
-import tifffile
from typing import Tuple
+from tifffile import TiffFile
+
from .. import types
from ..vendor import omexml
-from .reader import Reader
from .tiff_reader import TiffReader
+###############################################################################
+
log = logging.getLogger(__name__)
+###############################################################################
+
-class OmeTiffReader(Reader):
- """Opening and processing the contents of an OME Tiff file
+class OmeTiffReader(TiffReader):
+ """
+ Opening and processing the contents of an OME Tiff file
"""
- def __init__(self, file: types.FileLike, **kwargs):
- super().__init__(file, **kwargs)
- try:
- self.tiff = tifffile.TiffFile(self._bytes)
- except Exception:
- log.error("tiffile could not parse this input")
- raise
+ def __init__(self, data: types.FileLike, **kwargs):
+ super().__init__(data, **kwargs)
- def _lazy_init_metadata(self) -> omexml.OMEXML:
- if self._metadata is None and self.tiff.is_ome:
- description = self.tiff.pages[0].description.strip()
- if not (description.startswith("")):
- raise ValueError(f'Description does not conform to OME specification: {description[:100]}')
- self._metadata = omexml.OMEXML(description)
- return self._metadata
+ # Lazy load is ome
+ self._is_ome = None
@staticmethod
- def _is_this_type(buffer: io.BufferedIOBase) -> bool:
+ def _is_this_type(buffer: io.BytesIO) -> bool:
is_tif = TiffReader._is_this_type(buffer)
if is_tif:
buf = TiffReader.get_image_description(buffer)
@@ -52,57 +48,42 @@ def _is_this_type(buffer: io.BufferedIOBase) -> bool:
return True
return False
- @property
- def data(self) -> np.ndarray:
- if self._data is None:
- # load the data
- self._data = self.tiff.asarray()
- return self._data
+ def _lazy_init_metadata(self) -> omexml.OMEXML:
+ with TiffFile(self._file) as tiff:
+ if self._metadata is None and tiff.is_ome:
+ description = tiff.pages[0].description.strip()
+ if not (description.startswith("")):
+ raise ValueError(f"Description does not conform to OME specification: {description[:100]}")
+ self._metadata = omexml.OMEXML(description)
+ return self._metadata
- @property
- def dims(self) -> str:
- self._lazy_init_metadata()
- dimension_order = self._metadata.image().Pixels.DimensionOrder
- # reverse the string
- dimension_order = dimension_order[::-1]
- # see if t,z,or c is squeezed out.
- # this is a tifffile implementation detail -- see squeeze_axes in tifffile.
- if self.size_t() < 2:
- dimension_order = dimension_order.replace("T", "")
- if self.size_z() < 2:
- dimension_order = dimension_order.replace("Z", "")
- if self.size_c() < 2:
- dimension_order = dimension_order.replace("C", "")
- return dimension_order
+ def is_ome(self):
+ return self.is_this_type(self._file)
@property
def metadata(self) -> omexml.OMEXML:
- return self._lazy_init_metadata()
-
- def load_slice(self, slice_index=0):
- data = self.tiff.asarray(key=slice_index)
- return data
+ if self._metadata is None:
+ return self._lazy_init_metadata()
- def size_z(self):
- return self.metadata.image().Pixels.SizeZ
+ return self._metadata
- def size_c(self):
- return self.metadata.image().Pixels.SizeC
+ def size_s(self) -> int:
+ return self.metadata.image_count
- def size_t(self):
+ def size_t(self) -> int:
return self.metadata.image().Pixels.SizeT
- def size_x(self):
- return self.metadata.image().Pixels.SizeX
+ def size_c(self) -> int:
+ return self.metadata.image().Pixels.SizeC
- def size_y(self):
- return self.metadata.image().Pixels.SizeY
+ def size_z(self) -> int:
+ return self.metadata.image().Pixels.SizeZ
- def dtype(self):
- return self.tiff.pages[0].dtype
+ def size_y(self) -> int:
+ return self.metadata.image().Pixels.SizeY
- def is_ome(self):
- return OmeTiffReader._is_this_type(self._bytes)
+ def size_x(self) -> int:
+ return self.metadata.image().Pixels.SizeX
def get_channel_names(self, scene: int = 0):
return self.metadata.image(scene).Pixels.get_channel_names()
diff --git a/aicsimageio/readers/reader.py b/aicsimageio/readers/reader.py
index 4a50ccf52..709a34dab 100644
--- a/aicsimageio/readers/reader.py
+++ b/aicsimageio/readers/reader.py
@@ -2,77 +2,126 @@
# -*- coding: utf-8 -*-
import io
+import logging
from abc import ABC, abstractmethod
from pathlib import Path
-from typing import Any
+from typing import Any, Dict, Optional, Tuple, Union
+import dask.array as da
import numpy as np
+from distributed import Client, LocalCluster
-from .. import constants, types
+from .. import dask_utils, exceptions, types
+from ..constants import Dimensions
+###############################################################################
-class Reader(ABC):
+log = logging.getLogger(__name__)
+
+###############################################################################
- _bytes = None
+class Reader(ABC):
+ _dask_data = None
_data = None
_dims = None
_metadata = None
- def __init__(self, file: types.FileLike, **kwargs):
- # Convert to BytesIO
- self._bytes = self.convert_to_buffer(file)
-
- @staticmethod
- def guess_dim_order(shape: tuple) -> str:
- return constants.DEFAULT_DIMENSION_ORDER[len(constants.DEFAULT_DIMENSION_ORDER) - len(shape):]
-
@staticmethod
- def convert_to_buffer(file: types.FileLike) -> io.BufferedIOBase:
- # Check path
- if isinstance(file, (str, Path)):
- # This will both fully expand and enforce that the filepath exists
- f = Path(file).expanduser().resolve(strict=True)
+ def _resolve_image_path(img: Union[str, Path]) -> Path:
+ # Convert pathlike to Path
+ if isinstance(img, (str, Path)):
+ # Strictly do not fully resolve the path because Mac is bad with mounted drives
+ img = Path(img).expanduser()
+
+ # Check the file exists
+ if not img.exists():
+ raise FileNotFoundError(img)
+
+ # Check path
+ if img.is_dir():
+ raise IsADirectoryError(
+ f"Please provide a single file to the `img` parameter. "
+ f"Received directory: {img}"
+ )
+
+ # Check that no other type was provided
+ if not isinstance(img, Path):
+ raise TypeError(
+ f"Please provide a path to a file as a string, or an pathlib.Path, to the "
+ f"`img` parameter. "
+ f"Received type: {type(img)}"
+ )
- # This will check if the above enforced filepath is a directory
- if f.is_dir():
- raise IsADirectoryError(f)
+ return img
- return open(f, "rb")
+ def __init__(self, file: types.ImageLike, dask_kwargs: Dict[str, Any] = {}, **kwargs):
+ # This will both fully expand and enforce that the filepath exists
+ file = self._resolve_image_path(file)
- # Convert bytes
- elif isinstance(file, bytes):
- return io.BytesIO(file)
+ # Check type
+ if not self.is_this_type(file):
+ raise exceptions.UnsupportedFileFormatError(
+ f"Reader does not support file or object: {file}"
+ )
- # Set bytes
- elif isinstance(file, io.BytesIO):
- return file
+ # Store this filepath
+ self._file = file
- # Special case for ndarray because already in memory
- elif isinstance(file, np.ndarray):
- return file
+ # Store dask client and cluster setup
+ self._dask_kwargs = dask_kwargs
+ self._client = None
+ self._cluster = None
- # Raise
- else:
- raise TypeError(
- f"Reader only accepts types: [str, pathlib.Path, bytes, io.BytesIO], received: {type(file)}"
- )
+ @staticmethod
+ def guess_dim_order(shape: Tuple[int]) -> str:
+ return Dimensions.DefaultOrder[len(Dimensions.DefaultOrder) - len(shape):]
@classmethod
- def is_this_type(cls, file: types.FileLike) -> bool:
- buffer = cls.convert_to_buffer(file)
- return cls._is_this_type(buffer)
+ def is_this_type(cls, data: types.ImageLike) -> bool:
+ # Check path
+ if isinstance(data, (str, Path)):
+ # Resolve image path
+ f = cls._resolve_image_path(data)
+
+ # Return and close the open pointer
+ with open(f, "rb") as read_bytes:
+ return cls._is_this_type(read_bytes)
+
+ # Convert bytes to BytesIO
+ if isinstance(data, bytes):
+ data = io.BytesIO(data)
+
+ # Check type
+ if isinstance(data, io.BytesIO):
+ return cls._is_this_type(data)
+
+ # Special cases
+ if isinstance(data, (np.ndarray, da.core.Array)):
+ return cls._is_this_type(data)
+
+ # Raise because none of the above returned
+ raise TypeError(
+ f"Reader only accepts types: [str, pathlib.Path, bytes, io.BytesIO, numpy or dask array]. "
+ f"Received: {type(data)}"
+ )
@staticmethod
@abstractmethod
- def _is_this_type(buffer: io.BufferedIOBase) -> bool:
+ def _is_this_type(buffer: io.BytesIO) -> bool:
pass
@property
@abstractmethod
- def data(self) -> np.ndarray:
+ def dask_data(self) -> da.core.Array:
pass
+ @property
+ def data(self) -> np.ndarray:
+ if self._data is None:
+ self._data = self.dask_data.compute()
+ return self._data
+
@property
@abstractmethod
def dims(self) -> str:
@@ -83,14 +132,34 @@ def dims(self) -> str:
def metadata(self) -> Any:
pass
- def load(self) -> types.LoadResults:
- return types.LoadResults(self.data, self.dims, self.metadata)
+ @property
+ def cluster(self) -> Optional[LocalCluster]:
+ return self._cluster
- def close(self) -> None:
- self._bytes.close()
+ @property
+ def client(self) -> Optional[Client]:
+ return self._client
+
+ def close(self):
+ """
+ Always close the Dask Client connection.
+ If connected to *strictly* a LocalCluster, close it down as well.
+ """
+ self._cluster, self._client = dask_utils.shutdown_cluster_and_client(self.cluster, self.client)
def __enter__(self):
+ """
+ If provided an address, create a Dask Client connection.
+ If not provided an address, create a LocalCluster and Client connection.
+ If not provided an address, other Dask kwargs are accepted and passed down to the LocalCluster object.
+ """
+ self._cluster, self._client = dask_utils.spawn_cluster_and_client(**self._dask_kwargs)
+
return self
def __exit__(self, exc_type, exc_val, exc_tb):
+ """
+ Always close the Dask Client connection.
+ If connected to *strictly* a LocalCluster, close it down as well.
+ """
self.close()
diff --git a/aicsimageio/readers/tiff_reader.py b/aicsimageio/readers/tiff_reader.py
index b24c6ea50..32a73c370 100644
--- a/aicsimageio/readers/tiff_reader.py
+++ b/aicsimageio/readers/tiff_reader.py
@@ -1,19 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import io
+import logging
+from pathlib import Path
from typing import Optional
+import dask.array as da
import numpy as np
-import tifffile
+from dask import delayed
+from tifffile import TiffFile
-from .. import types
+from .. import exceptions, types
from ..buffer_reader import BufferReader
-from . import reader
+from ..constants import Dimensions
+from .reader import Reader
+
+###############################################################################
+log = logging.getLogger(__name__)
-class TiffReader(reader.Reader):
- """This class is used to open and process the contents of a generic tiff file.
+###############################################################################
- The load function will get a 3D ZYX array from a tiff file.
+
+class TiffReader(Reader):
"""
+ TiffReader wraps tifffile to provide the same reading capabilities but abstracts the specifics of using the
+ backend library to create a unified interface. This enables higher level functions to duck type the File Readers.
+
+ Parameters
+ ----------
+ data: types.FileLike
+ A string or path to the TIFF file to be read.
+ S: int
+ If the image has different dimensions on any scene from another, the dask array construction will fail.
+ In that case, use this parameter to specify a specific scene to construct a dask array for.
+ Default: 0 (select the first scene)
+ """
+
+ def __init__(self, data: types.FileLike, S: int = 0, **kwargs):
+ # Run super init to check filepath provided
+ super().__init__(data, **kwargs)
+
+ # Store parameters needed for dask read
+ self.specific_s_index = S
+
+ # Lazy load and hold on to dtype
+ self._dtype = None
@staticmethod
def _is_this_type(buffer: io.BufferedIOBase) -> bool:
@@ -42,6 +75,158 @@ def _is_this_type(buffer: io.BufferedIOBase) -> bool:
return False
return True
+ @staticmethod
+ def _imread(
+ img: Path,
+ scene: int,
+ page: int
+ ) -> np.ndarray:
+ # Load Tiff
+ with TiffFile(img) as tiff:
+ # Get proper scene
+ scene = tiff.series[scene]
+
+ # Get proper page
+ page = scene.pages[page]
+
+ # Return numpy
+ return page.asarray()
+
+ @property
+ def dask_data(self) -> da.core.Array:
+ """
+ Read a TIFF image file as a delayed dask array where each chunk of the constructed array is a delayed YX plane.
+
+ Returns
+ -------
+ img: dask.array.core.Array
+ The constructed delayed YX plane dask array.
+ """
+ if self._dask_data is None:
+ # Load Tiff
+ with TiffFile(self._file) as tiff:
+ # Check each scene has the same shape
+ # If scene shape checking fails, use the specified scene and update operating shape
+ scenes = tiff.series
+ operating_shape = scenes[0].shape
+ for scene in scenes:
+ if scene.shape != operating_shape:
+ operating_shape = scenes[self.specific_s_index].shape
+ scenes = [scenes[self.specific_s_index]]
+ log.info(
+ f"File contains variable dimensions per scene, "
+ f"selected scene: {self.specific_s_index} for data retrieval."
+ )
+ break
+
+ # Get sample yx plane
+ sample = scenes[0].pages[0].asarray()
+
+ # Combine length of scenes and operating shape
+ # Replace YX dims with empty dimensions
+ operating_shape = (len(scenes), *operating_shape)
+ operating_shape = operating_shape[:-2] + (1, 1)
+
+ # Make ndarray for lazy arrays to fill
+ lazy_arrays = np.ndarray(operating_shape, dtype=object)
+ for all_page_index, (np_index, _) in enumerate(np.ndenumerate(lazy_arrays)):
+ # Scene index is the first index in np_index
+ scene_index = np_index[0]
+
+ # This page index is current enumeration divided by scene index + 1
+ # For example if the image has 10 Z slices and 5 scenes, there would be 50 total pages
+ this_page_index = all_page_index // (scene_index + 1)
+
+ # Fill the numpy array with the delayed arrays
+ lazy_arrays[np_index] = da.from_delayed(
+ delayed(TiffReader._imread)(self._file, scene_index, this_page_index),
+ shape=sample.shape,
+ dtype=sample.dtype
+ )
+
+ # Convert the numpy array of lazy readers into a dask array
+ data = da.block(lazy_arrays.tolist())
+
+ # Only return the scene dimension if multiple scenes are present
+ if len(scenes) == 1:
+ data = data[0, :]
+
+ # Set _dask_data
+ self._dask_data = data
+
+ return self._dask_data
+
+ def load_slice(self, slice_index: int = 0) -> np.ndarray:
+ with TiffFile(self._file) as tiff:
+ return tiff.asarray(key=slice_index)
+
+ def dtype(self):
+ if self._dtype is None:
+ with TiffFile(self._file) as tiff:
+ self._dtype = tiff.pages[0].dtype
+
+ return self._dtype
+
+ @property
+ def dims(self) -> str:
+ if self._dims is None:
+ # Get a single scenes dimensions in order
+ with TiffFile(self._file) as tiff:
+ single_scene_dims = tiff.series[0].pages.axes
+
+ # We can sometimes trust the dimension info in the image
+ if all([d in Dimensions.DefaultOrder for d in single_scene_dims]):
+ # Add scene dimension only if there are multiple scenes
+ if len(tiff.series) == 1:
+ self._dims = single_scene_dims
+ else:
+ self._dims = f"{Dimensions.Scene}{single_scene_dims}"
+ # Sometimes the dimension info is wrong in certain dimensions, so guess that dimension
+ else:
+ guess = self.guess_dim_order(tiff.series[0].pages.shape)
+ best_guess = []
+ for dim_from_meta in single_scene_dims:
+ if dim_from_meta in Dimensions.DefaultOrder:
+ best_guess.append(dim_from_meta)
+ else:
+ appended_dim = False
+ for guessed_dim in guess:
+ if guessed_dim not in best_guess:
+ best_guess.append(guessed_dim)
+ appended_dim = True
+ log.info(
+ f"Unsure how to handle dimension: {dim_from_meta}. "
+ f"Replaced with guess: {guessed_dim}"
+ )
+ break
+
+ # All of our guess dims were already in the dim list, append the dim read from meta
+ if not appended_dim:
+ best_guess.append(dim_from_meta)
+
+ best_guess = "".join(best_guess)
+
+ # Add scene dimension only if there are multiple scenes
+ if len(tiff.series) == 1:
+ self._dims = best_guess
+ else:
+ self._dims = f"{Dimensions.Scene}{best_guess}"
+
+ return self._dims
+
+ @dims.setter
+ def dims(self, dims: str):
+ # Check amount of provided dims against data shape
+ if len(dims) != len(self.dask_data.shape):
+ raise exceptions.InvalidDimensionOrderingError(
+ f"Provided too many dimensions for the associated file. "
+ f"Received {len(dims)} dimensions [dims: {dims}] "
+ f"for image with {len(self.data.shape)} dimensions [shape: {self.data.shape}]."
+ )
+
+ # Set the dims
+ self._dims = dims
+
@staticmethod
def get_image_description(buffer: io.BufferedIOBase) -> Optional[bytearray]:
"""Retrieve the image description as one large string."""
@@ -107,40 +292,13 @@ def get_image_description(buffer: io.BufferedIOBase) -> Optional[bytearray]:
buffer_reader.buffer.seek(description_offset, 0)
return bytearray(buffer_reader.buffer.read(description_length))
- def __init__(self, file: types.FileLike, **kwargs):
- super().__init__(file, **kwargs)
- self.tiff = tifffile.TiffFile(self._bytes)
-
- def close(self):
- self.tiff.close()
- super().close()
-
- def dtype(self):
- return self.tiff.pages[0].dtype
-
- @property
- def data(self) -> np.ndarray:
- if self._data is None:
- self._data = self.tiff.asarray()
- return self._data
-
- @property
- def dims(self) -> str:
- if self._dims is None:
- self._dims = self.guess_dim_order(self.data.shape)
-
- return self._dims
-
- @dims.setter
- def dims(self, value: str) -> None:
- self._dims = value
-
@property
def metadata(self) -> str:
if self._metadata is None:
- description = self.get_image_description(self._bytes)
+ with open(self._file, "rb") as rb:
+ description = self.get_image_description(rb)
if description is None:
- self._metadata = ''
+ self._metadata = ""
else:
self._metadata = description.decode()
return self._metadata
diff --git a/aicsimageio/tests/readers/__init__.py b/aicsimageio/tests/readers/__init__.py
index e69de29bb..faa18be5b 100644
--- a/aicsimageio/tests/readers/__init__.py
+++ b/aicsimageio/tests/readers/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
diff --git a/aicsimageio/tests/readers/test_arraylike_reader.py b/aicsimageio/tests/readers/test_arraylike_reader.py
new file mode 100644
index 000000000..b2e5e30f2
--- /dev/null
+++ b/aicsimageio/tests/readers/test_arraylike_reader.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import dask.array as da
+import numpy as np
+import pytest
+from dask.diagnostics import Profiler
+
+from aicsimageio import exceptions
+from aicsimageio.readers import ArrayLikeReader
+
+
+@pytest.mark.parametrize("arr, expected_shape, expected_dims, expected_chunksize, expected_task_count", [
+ (np.ones((1, 1)), (1, 1), "YX", (1, 1), 0),
+ (np.ones((1, 1, 1)), (1, 1, 1), "ZYX", (1, 1, 1), 0),
+ (np.ones((1, 1, 1, 1)), (1, 1, 1, 1), "CZYX", (1, 1, 1, 1), 0),
+ (da.ones((1, 1)), (1, 1), "YX", (1, 1), 1),
+ (da.ones((1, 1, 1)), (1, 1, 1), "ZYX", (1, 1, 1), 1),
+ (da.ones((1, 1, 1, 1)), (1, 1, 1, 1), "CZYX", (1, 1, 1, 1), 1),
+ pytest.param(
+ "hello_word",
+ None,
+ None,
+ None,
+ None,
+ marks=pytest.mark.raises(exceptions=TypeError)
+ )
+])
+def test_arraylike_reader(arr, expected_shape, expected_dims, expected_chunksize, expected_task_count):
+ # Init
+ reader = ArrayLikeReader(arr)
+
+ # Check basics
+ with Profiler() as prof:
+ assert reader.dims == expected_dims
+ assert reader.metadata is None
+ assert reader.dask_data.shape == expected_shape
+ assert reader.dask_data.chunksize == expected_chunksize
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check computed type is numpy array, computed shape is expected shape, and task count is expected
+ with Profiler() as prof:
+ assert isinstance(reader.data, np.ndarray)
+ assert reader.data.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+
+@pytest.mark.parametrize("expected_starting_dims, set_dims, expected_ending_dims", [
+ ("ZYX", "YXC", "YXC"),
+ ("ZYX", "TYX", "TYX"),
+ ("ZYX", "ABC", "ABC"),
+ pytest.param("ZYX", "ABCDE", None, marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError))
+])
+def test_dims_setting(expected_starting_dims, set_dims, expected_ending_dims):
+ # Read file
+ img = ArrayLikeReader(da.ones((2, 2, 2)))
+
+ # Check basics
+ with Profiler() as prof:
+ assert img.dims == expected_starting_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check no tasks happen during dims setting
+ with Profiler() as prof:
+ img.dims = set_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check no tasks happen during dims getting
+ with Profiler() as prof:
+ assert img.dims == expected_ending_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
diff --git a/aicsimageio/tests/readers/test_czi_reader.py b/aicsimageio/tests/readers/test_czi_reader.py
index 13f6485ba..b5c62de4d 100644
--- a/aicsimageio/tests/readers/test_czi_reader.py
+++ b/aicsimageio/tests/readers/test_czi_reader.py
@@ -1,118 +1,223 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
from io import BytesIO
-from pathlib import Path
+import numpy as np
import pytest
+from dask.diagnostics import Profiler
+from psutil import Process
-from aicsimageio.exceptions import UnsupportedFileFormatError
+from aicsimageio import exceptions
from aicsimageio.readers.czi_reader import CziReader
-# Example files
-TWO_DIM_CZI = "s_1_t_1_c_1_z_1.czi"
-SIX_DIM_CZI = "s_3_t_1_c_3_z_5.czi"
-
-
-# TODO It would be good to test that given a multiscene defined exception is raised
-
-@pytest.mark.parametrize("file", [
- TWO_DIM_CZI,
- SIX_DIM_CZI,
- pytest.param(BytesIO(b"abcdef"), marks=pytest.mark.raises(exception=UnsupportedFileFormatError)),
- pytest.param("non_existent_file.random", marks=pytest.mark.raises(exception=FileNotFoundError)),
- pytest.param(Path("/nonexistent/file/file.random"), marks=pytest.mark.raises(exception=FileNotFoundError))
+@pytest.mark.parametrize(
+ "filename, "
+ "expected_shape, "
+ "expected_dims, "
+ "expected_dtype, "
+ "select_scene, "
+ "chunk_dims, "
+ "expected_chunksize, "
+ "expected_task_count",
+ [
+ # Expected task counts should be each non chunk dimension size multiplied againest each other * 2
+ (
+ "s_1_t_1_c_1_z_1.czi",
+ (1, 1, 325, 475),
+ "BCYX",
+ np.uint16,
+ 0,
+ ("Z", "Y", "X"),
+ (1, 1, 325, 475),
+ 2 # 1 * 1 * 2 = 2
+ ),
+ (
+ "s_3_t_1_c_3_z_5.czi",
+ (1, 3, 3, 5, 325, 475),
+ "BSCZYX",
+ np.uint16,
+ 0,
+ ("Z", "Y", "X"),
+ (1, 1, 1, 5, 325, 475),
+ 18 # 1 * 3 * 3 * 2 = 18
+ ),
+ (
+ "s_3_t_1_c_3_z_5.czi",
+ (1, 3, 3, 5, 325, 475),
+ "BSCZYX",
+ np.uint16,
+ 0,
+ ("Y", "X"),
+ (1, 1, 1, 1, 325, 475),
+ 90 # 1 * 3 * 3 * 5 * 2 = 90
+ ),
+ (
+ "s_3_t_1_c_3_z_5.czi",
+ (1, 3, 3, 5, 325, 475),
+ "BSCZYX",
+ np.uint16,
+ 0,
+ ("C", "Y", "X"),
+ (1, 1, 3, 1, 325, 475),
+ 30 # 1 * 3 * 5 * 2 = 30
+ ),
+ (
+ "s_3_t_1_c_3_z_5.czi",
+ (1, 3, 3, 5, 325, 475),
+ "BSCZYX",
+ np.uint16,
+ 0,
+ ("S", "Y", "X"),
+ (1, 3, 1, 1, 325, 475),
+ 30 # 1 * 3 * 5 * 2 = 30
+ ),
+ # Check that Spatial Y and Spatial X dims are always added to chunk dims
+ (
+ "s_3_t_1_c_3_z_5.czi",
+ (1, 3, 3, 5, 325, 475),
+ "BSCZYX",
+ np.uint16,
+ 0,
+ ("S"),
+ (1, 3, 1, 1, 325, 475),
+ 30 # 1 * 3 * 5 * 2 = 30
+ ),
+ (
+ "variable_per_scene_dims.czi",
+ (1, 1, 2, 1, 2, 1248, 1848),
+ "BSTCZYX",
+ np.uint16,
+ 0,
+ ("Z", "Y", "X"),
+ (1, 1, 1, 1, 2, 1248, 1848),
+ 4 # 1 * 1 * 2 * 1 * 2 = 4
+ ),
+ (
+ "variable_per_scene_dims.czi",
+ (1, 1, 1, 1, 2, 1248, 1848),
+ "BSTCZYX",
+ np.uint16,
+ 1,
+ ("Z", "Y", "X"),
+ (1, 1, 1, 1, 2, 1248, 1848),
+ 2 # 1 * 1 * 1 * 1 * 2 = 2
+ ),
+ pytest.param(
+ "variable_per_scene_dims.czi",
+ (1, 1, 1, 1, 2, 1248, 1848),
+ "BSTCZYX",
+ np.uint16,
+ 2,
+ ("Z", "Y", "X"),
+ None,
+ None, # 1 * 1 * 1 * 1 * 2 = 2,
+ marks=pytest.mark.raises(exception=exceptions.InconsistentShapeError)
+ ),
+ pytest.param(
+ "example.txt",
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError)
+ )
]
)
-def test_czi_reader(resources_dir, file):
- fobj = file
- if isinstance(file, str):
- fobj = resources_dir / file
- czi = CziReader(fobj)
- czi.close()
-
-
-@pytest.mark.parametrize("test_input, expected", [
+def test_czi_reader(
+ resources_dir,
+ filename,
+ expected_shape,
+ expected_dims,
+ expected_dtype,
+ select_scene,
+ chunk_dims,
+ expected_chunksize,
+ expected_task_count
+):
+ # Get file
+ f = resources_dir / filename
+
+ # Read file
+ img = CziReader(f, chunk_by_dims=chunk_dims, S=select_scene)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ assert img.dims == expected_dims
+ assert img.metadata
+ assert img.dask_data.shape == expected_shape
+ assert img.dask_data.chunksize == expected_chunksize
+ assert img.dtype() == expected_dtype
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check computed type is numpy array, computed shape is expected shape, and task count is expected
+ with Profiler() as prof:
+ assert isinstance(img.data, np.ndarray)
+ assert img.data.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+ # Check that there are no open file pointers after retrieval
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("raw_bytes, expected", [
(BytesIO(b"ZInotaczinope"), False),
(BytesIO(b"ZISRAWFILE"), True),
(BytesIO(b"ZI"), False),
(BytesIO(b""), False)
- ]
-)
-def test_is_this_type(test_input, expected):
- res = CziReader._is_this_type(test_input)
- assert res == expected
-
-
-@pytest.mark.parametrize("test_input, expected", [
- (TWO_DIM_CZI, "BCYX0"),
- (SIX_DIM_CZI, "BSCZYX0")
])
-def test_dims(resources_dir, test_input, expected):
- czi = CziReader(resources_dir / test_input)
- assert czi.dims == expected
+def test_is_this_type(raw_bytes, expected):
+ res = CziReader._is_this_type(raw_bytes)
+ assert res == expected
-@pytest.mark.parametrize("test_input, expected", [
- (TWO_DIM_CZI, "uint16"),
- (SIX_DIM_CZI, "uint16")
+@pytest.mark.parametrize("filename, scene, expected", [
+ ("s_1_t_1_c_1_z_1.czi", 0, ["Bright"]),
+ ("s_3_t_1_c_3_z_5.czi", 0, ["EGFP", "TaRFP", "Bright"]),
+ ("s_3_t_1_c_3_z_5.czi", 1, ["EGFP", "TaRFP", "Bright"]),
+ ("s_3_t_1_c_3_z_5.czi", 2, ["EGFP", "TaRFP", "Bright"]),
+ # Our current get channel names doesn't take scene into account
+ # pytest.param("s_3_t_1_c_3_z_5.czi", 3, None, marks=pytest.mark.raises(exception=IndexError))
])
-def test_dtype(resources_dir, test_input, expected):
- czi = CziReader(resources_dir / test_input)
- assert czi.dtype() == expected
+def test_get_channel_names(resources_dir, filename, scene, expected):
+ assert CziReader(resources_dir / filename).get_channel_names(scene) == expected
-@pytest.mark.parametrize("test_input, expected", [
- (TWO_DIM_CZI, (1.0833333333333333e-06, 1.0833333333333333e-06, 1.0)),
- (SIX_DIM_CZI, (1.0833333333333333e-06, 1.0833333333333333e-06, 1e-06))
+@pytest.mark.parametrize("filename, expected", [
+ ("s_1_t_1_c_1_z_1.czi", (1.0833333333333333e-06, 1.0833333333333333e-06, 1.0)),
+ ("s_3_t_1_c_3_z_5.czi", (1.0833333333333333e-06, 1.0833333333333333e-06, 1e-06))
])
-def test_pixel_size(resources_dir, test_input, expected):
- czi = CziReader(resources_dir / test_input)
- assert czi.get_physical_pixel_size() == expected
-
-
-def test_shape(resources_dir):
- czi = CziReader(resources_dir / SIX_DIM_CZI)
- data = czi.data
- data_shape = data.shape
- # BSCZYX0
- assert data_shape[0] == 1
- assert data_shape[1] == czi.size_s()
- assert data_shape[2] == czi.size_c()
- assert data_shape[3] == czi.size_z()
- assert data_shape[4] == czi.size_y()
- assert data_shape[5] == czi.size_x()
- assert czi._is_multiscene() is False
-
-
-@pytest.mark.parametrize("test_input, expected", [
- ("T", -1), ("C", 2), ("Z", 3), ("Y", 4), ("X", 5), ("S", 1), ("B", 0), ("M", -1), ("V", -1),
- pytest.param("TZ", False, marks=pytest.mark.raises(exception=TypeError))
-])
-def test_dimension_index(resources_dir, test_input, expected):
- czi = CziReader(resources_dir / SIX_DIM_CZI)
- # BSCZYX0
- assert czi._lookup_dimension_index(test_input) == expected
-
-
-def test_missing_dimension(resources_dir):
- czi = CziReader(resources_dir / SIX_DIM_CZI)
- assert czi._size_of_dimension("V") == 1
- assert czi._lookup_dimension_index("V") == -1
+def test_get_physical_pixel_size(resources_dir, filename, expected):
+ assert CziReader(resources_dir / filename).get_physical_pixel_size() == expected
-# NOTE:
-# These are all going to have the same channel setup id because they were all created from the same base file
-# As in, look at the most complex czi, and all of the other czis are just various slices of that one.
-@pytest.mark.parametrize("test_input, expected", [
- (TWO_DIM_CZI, "636972569326165806"),
- (SIX_DIM_CZI, "636972569326165806")
+@pytest.mark.parametrize("filename, s, t, c, z, y, x", [
+ ("s_1_t_1_c_1_z_1.czi", 1, 1, 1, 1, 325, 475),
+ ("s_3_t_1_c_3_z_5.czi", 3, 1, 3, 5, 325, 475)
])
-def test_metadata(resources_dir, test_input, expected):
- czi = CziReader(resources_dir / test_input)
- checked = False
- for it in czi.metadata.iter("Channel"):
- x = it.attrib.get("ChannelSetupId")
- if x:
- assert x == expected
- checked = True
- break
- assert checked
+def test_size_functions(resources_dir, filename, s, t, c, z, y, x):
+ # Get file
+ f = resources_dir / filename
+
+ # Init reader
+ img = CziReader(f)
+
+ # Check sizes
+ assert img.size_s() == s
+ assert img.size_t() == t
+ assert img.size_c() == c
+ assert img.size_z() == z
+ assert img.size_y() == y
+ assert img.size_x() == x
diff --git a/aicsimageio/tests/readers/test_default_reader.py b/aicsimageio/tests/readers/test_default_reader.py
index 7acb52c1b..f138a1a8f 100644
--- a/aicsimageio/tests/readers/test_default_reader.py
+++ b/aicsimageio/tests/readers/test_default_reader.py
@@ -1,37 +1,110 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+import numpy as np
import pytest
+from dask.diagnostics import Profiler
+from psutil import Process
from aicsimageio import exceptions
from aicsimageio.readers.default_reader import DefaultReader
-@pytest.mark.parametrize("filename", [
- "example.bmp",
- "example.png",
- "example.jpg",
- "example.gif"
+@pytest.mark.parametrize("filename, expected_shape, expected_dims, expected_chunksize, expected_task_count", [
+ ("example.bmp", (480, 640, 4), "YXC", (480, 640, 4), 0),
+ ("example.png", (800, 537, 4), "YXC", (800, 537, 4), 0),
+ ("example.jpg", (452, 400, 3), "YXC", (452, 400, 3), 0),
+ # Task count for multiple image formats should be 2 * number of images in file
+ ("example.gif", (72, 268, 268, 4), "TYXC", (1, 268, 268, 4), 144),
+ pytest.param(
+ "example.txt",
+ None,
+ None,
+ None,
+ None,
+ marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError)
+ )
])
-def test_default_reader_get_default_dims(resources_dir, filename):
+def test_default_reader(
+ resources_dir,
+ filename,
+ expected_shape,
+ expected_dims,
+ expected_chunksize,
+ expected_task_count
+):
# Get file
f = resources_dir / filename
- # Open
- with DefaultReader(f) as r:
- # Dims should be set to 3D for all of these images
- assert r.dims == "YXC"
- assert r.metadata is None
- assert DefaultReader.is_this_type(f)
+ # Read file
+ img = DefaultReader(f)
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
-@pytest.mark.parametrize("expected", [
- "XYC",
- "STC",
- pytest.param("HELLOWORLD", marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError)),
- pytest.param("NO", marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError))
+ # Check basics
+ with Profiler() as prof:
+ assert img.dims == expected_dims
+ assert img.metadata
+ assert img.dask_data.shape == expected_shape
+ assert img.dask_data.chunksize == expected_chunksize
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check computed type is numpy array, computed shape is expected shape, and task count is expected
+ with Profiler() as prof:
+ assert isinstance(img.data, np.ndarray)
+ assert img.data.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+ # Check that there are no open file pointers after retrieval
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("expected_starting_dims, set_dims, expected_ending_dims", [
+ ("YXC", "ZXC", "ZXC"),
+ ("YXC", "YXZ", "YXZ"),
+ ("YXC", "ABC", "ABC"),
+ pytest.param("YXC", "ABCDE", None, marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError))
])
-def test_default_reader_set_dims(resources_dir, expected):
- with DefaultReader(resources_dir / "example.png") as r:
- r.dims = expected
- assert r.dims == expected
+def test_dims_setting(resources_dir, expected_starting_dims, set_dims, expected_ending_dims):
+ # Get file
+ f = resources_dir / "example.png"
+
+ # Read file
+ img = DefaultReader(f)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ assert img.dims == expected_starting_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check no tasks happen during dims setting
+ with Profiler() as prof:
+ img.dims = set_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check no tasks happen during dims getting
+ with Profiler() as prof:
+ assert img.dims == expected_ending_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after retrieval
+ assert str(f) not in [f.path for f in proc.open_files()]
diff --git a/aicsimageio/tests/readers/test_ndarray_reader.py b/aicsimageio/tests/readers/test_ndarray_reader.py
deleted file mode 100644
index 174c1ec2f..000000000
--- a/aicsimageio/tests/readers/test_ndarray_reader.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import numpy as np
-import pytest
-from aicsimageio import exceptions
-from aicsimageio.readers.ndarray_reader import NdArrayReader
-
-
-@pytest.mark.parametrize("arr, expected_dims", [
- (np.ones((1, 1)), "YX"),
- (np.ones((1, 1, 1)), "ZYX"),
- (np.ones((1, 1, 1, 1)), "CZYX")
-])
-def test_ndarray_reader_get_default_dims(arr, expected_dims):
- # Open
- with NdArrayReader(arr) as r:
- assert r.dims == expected_dims
- assert r.metadata is None
- assert NdArrayReader.is_this_type(arr)
-
-
-@pytest.mark.parametrize("expected", [
- "XYC",
- "STC",
- pytest.param("HELLOWORLD", marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError)),
- pytest.param("NO", marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError))
-])
-def test_default_reader_set_dims(expected):
- with NdArrayReader(np.ones((1, 1, 1))) as r:
- r.dims = expected
- assert r.dims == expected
diff --git a/aicsimageio/tests/readers/test_ome_tiff_reader.py b/aicsimageio/tests/readers/test_ome_tiff_reader.py
index 94db080fe..e6b529e84 100644
--- a/aicsimageio/tests/readers/test_ome_tiff_reader.py
+++ b/aicsimageio/tests/readers/test_ome_tiff_reader.py
@@ -1,71 +1,192 @@
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
-import os
-import unittest
-
-from aicsimageio.readers import OmeTiffReader
-
-
-class TestOmeTifReader(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- cls.dir_path = os.path.dirname(os.path.realpath(__file__))
- with OmeTiffReader(
- os.path.join(cls.dir_path, "..", "resources", "s_1_t_1_c_1_z_1.ome.tiff")
- ) as reader:
- cls.load = reader.data
- cls.load_sizes = [
- reader.size_t(),
- reader.size_c(),
- reader.size_z(),
- reader.size_y(),
- reader.size_x(),
- ]
- cls.dims = reader.dims
- cls.metadata = reader.metadata
-
- def test_omeTifLoadShapeCorrectDimensions(self):
- self.assertEqual(len(self.load.shape), 2)
-
- def test_omeTifEmptyFileError(self):
- with self.assertRaises(Exception):
- with OmeTiffReader("fakefile") as reader:
- assert reader.data
-
- def test_notOmeTifFile(self):
- with self.assertRaises(Exception):
- with OmeTiffReader(
- os.path.join(
- self.dir_path, "resources", "s_1_t_1_c_1_z_1.czi"
- )
- ) as reader:
- assert reader.data
-
- def test_loadSampleOmeTif(self):
- names = [
+import numpy as np
+import pytest
+from dask.diagnostics import Profiler
+from psutil import Process
+
+from aicsimageio import exceptions
+from aicsimageio.constants import Dimensions
+from aicsimageio.readers.ome_tiff_reader import OmeTiffReader
+
+
+@pytest.mark.parametrize(
+ "filename, "
+ "expected_shape, "
+ "expected_dims, "
+ "select_scene, "
+ "expected_chunksize, "
+ "expected_task_count",
+ [
+ # Expected task counts should be each non chunk dimension size multiplied againest each other * 2
+ (
"s_1_t_1_c_1_z_1.ome.tiff",
- "s_1_t_1_c_10_z_1.ome.tiff",
- "s_3_t_1_c_3_z_5.ome.tiff"
- ]
- dims = [
(325, 475),
- (10, 1736, 1776),
- (5, 3, 325, 475),
- ]
- dim_orders = [
"YX",
- "CYX", # Inferred from metadata not shape
- "ZCYX",
- ]
- physical_pixel_sizes = [
- (1.0833333333333333, 1.0833333333333333, 1.0),
- (1.0, 1.0, 1.0),
- (1.0833333333333333, 1.0833333333333333, 1.0),
- ]
- for i, x in enumerate(names):
- with OmeTiffReader(os.path.join(self.dir_path, "..", "resources", x)) as reader:
- assert reader.is_ome()
- data = reader.data
- self.assertEqual(data.shape, dims[i])
- self.assertEqual(reader.dims, dim_orders[i])
- self.assertEqual(reader.get_physical_pixel_size(), physical_pixel_sizes[i])
+ 0,
+ (325, 475),
+ 2 # 2 = 2
+ ),
+ (
+ "s_1_t_1_c_10_z_1.ome.tiff",
+ (10, 1736, 1776),
+ "CYX",
+ 0,
+ (1, 1736, 1776),
+ 20 # 10 * 2 = 2
+ ),
+ (
+ "s_3_t_1_c_3_z_5.ome.tiff",
+ (3, 5, 3, 325, 475),
+ "SZCYX",
+ 0,
+ (1, 1, 1, 325, 475),
+ 90 # 3 * 5 * 3 * 2 = 90
+ ),
+ pytest.param(
+ "example.txt",
+ None,
+ None,
+ None,
+ None,
+ None,
+ marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError)
+ ),
+ pytest.param(
+ "s_1_t_1_c_1_z_1.tiff",
+ None,
+ None,
+ None,
+ None,
+ None,
+ marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError)
+ )
+ ]
+)
+def test_ome_tiff_reader(
+ resources_dir,
+ filename,
+ expected_shape,
+ expected_dims,
+ select_scene,
+ expected_chunksize,
+ expected_task_count
+):
+ # Get file
+ f = resources_dir / filename
+
+ # Read file
+ img = OmeTiffReader(f, S=select_scene)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ # Check that OME Metadata matches the dask data array shape and dims order
+ dim_size_getters = {
+ Dimensions.Scene: img.size_s,
+ Dimensions.Time: img.size_t,
+ Dimensions.Channel: img.size_c,
+ Dimensions.SpatialZ: img.size_z,
+ Dimensions.SpatialY: img.size_y,
+ Dimensions.SpatialX: img.size_x
+ }
+ for d, getter in dim_size_getters.items():
+ if d in expected_dims:
+ assert getter() == img.dask_data.shape[img.dims.index(d)]
+
+ assert img.dims == expected_dims
+ assert img.is_ome()
+ assert img.metadata
+ assert img.dask_data.shape == expected_shape
+ assert img.dask_data.chunksize == expected_chunksize
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check computed type is numpy array, computed shape is expected shape, and task count is expected
+ with Profiler() as prof:
+ assert isinstance(img.data, np.ndarray)
+ assert img.data.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+ # Check that there are no open file pointers after retrieval
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("filename, scene, expected_channel_names", [
+ ("s_1_t_1_c_1_z_1.ome.tiff", 0, ["Bright"]),
+ ("s_1_t_1_c_10_z_1.ome.tiff", 0, [f"C:{i}" for i in range(10)]),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 0, ["EGFP", "TaRFP", "Bright"]),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 1, ["EGFP", "TaRFP", "Bright"]),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 2, ["EGFP", "TaRFP", "Bright"]),
+ pytest.param("s_1_t_1_c_1_z_1.ome.tiff", 1, None, marks=pytest.mark.raises(exception=IndexError))
+])
+def test_get_channel_names(resources_dir, filename, scene, expected_channel_names):
+ # Get file
+ f = resources_dir / filename
+
+ # Read file
+ img = OmeTiffReader(f)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check channel names
+ assert img.get_channel_names(scene) == expected_channel_names
+
+ # Check that there are no open file pointers after check
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("filename, scene, expected_pixel_sizes", [
+ ("s_1_t_1_c_1_z_1.ome.tiff", 0, (1.0833333333333333, 1.0833333333333333, 1.0)),
+ ("s_1_t_1_c_10_z_1.ome.tiff", 0, (1.0, 1.0, 1.0)),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 0, (1.0833333333333333, 1.0833333333333333, 1.0)),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 1, (1.0833333333333333, 1.0833333333333333, 1.0)),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 2, (1.0833333333333333, 1.0833333333333333, 1.0)),
+ pytest.param("s_1_t_1_c_1_z_1.ome.tiff", 1, None, marks=pytest.mark.raises(exception=IndexError))
+])
+def test_get_physical_pixel_size(resources_dir, filename, scene, expected_pixel_sizes):
+ # Get file
+ f = resources_dir / filename
+
+ # Read file
+ img = OmeTiffReader(f)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check physical pixel sizes
+ assert img.get_physical_pixel_size(scene) == expected_pixel_sizes
+
+ # Check that there are no open file pointers after check
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("filename, s, t, c, z, y, x", [
+ ("s_1_t_1_c_1_z_1.ome.tiff", 1, 1, 1, 1, 325, 475),
+ ("s_1_t_1_c_10_z_1.ome.tiff", 1, 1, 10, 1, 1736, 1776),
+ ("s_3_t_1_c_3_z_5.ome.tiff", 3, 1, 3, 5, 325, 475)
+])
+def test_size_functions(resources_dir, filename, s, t, c, z, y, x):
+ # Get file
+ f = resources_dir / filename
+
+ # Init reader
+ img = OmeTiffReader(f)
+
+ # Check sizes
+ assert img.size_s() == s
+ assert img.size_t() == t
+ assert img.size_c() == c
+ assert img.size_z() == z
+ assert img.size_y() == y
+ assert img.size_x() == x
diff --git a/aicsimageio/tests/readers/test_reader.py b/aicsimageio/tests/readers/test_reader.py
index 3c6c1ce8d..dfd061069 100644
--- a/aicsimageio/tests/readers/test_reader.py
+++ b/aicsimageio/tests/readers/test_reader.py
@@ -1,25 +1,25 @@
-from io import BytesIO
-from pathlib import Path
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import pytest
+
from aicsimageio.readers.reader import Reader
-@pytest.mark.parametrize("file", [
+@pytest.mark.parametrize("filename", [
+ ("example.png"),
+ pytest.param(None, marks=pytest.mark.raises(exception=IsADirectoryError)),
+ pytest.param(1, marks=pytest.mark.raises(exception=TypeError)),
pytest.param("non_existent_file.random", marks=pytest.mark.raises(exception=FileNotFoundError)),
- pytest.param(
- Path("/non/existent/file/path/non_existent_file.random"),
- marks=pytest.mark.raises(exception=FileNotFoundError)
- ),
- BytesIO(b"abcdef"),
- b"abcdef"
- ]
-)
-def test_reader_constructor(file):
- """
- Testing the arguments to the static member function on the ABC
- Parameters
- ----------
- file The various objects [str(filename), pathlib.Path, BytesIO, bytestring]
+])
+def test_resolve_image_path(resources_dir, filename):
+ # Get file
+ if isinstance(filename, str):
+ f = resources_dir / filename
+ elif filename is None:
+ f = resources_dir
+ else:
+ f = filename
- """
- Reader.convert_to_buffer(file)
+ # Test path resolution
+ Reader._resolve_image_path(f)
diff --git a/aicsimageio/tests/readers/test_tiff_reader.py b/aicsimageio/tests/readers/test_tiff_reader.py
index 993fa8248..f8714afd2 100644
--- a/aicsimageio/tests/readers/test_tiff_reader.py
+++ b/aicsimageio/tests/readers/test_tiff_reader.py
@@ -1,33 +1,163 @@
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import numpy as np
import pytest
+from dask.diagnostics import Profiler
+from psutil import Process
-from aicsimageio.readers import TiffReader
+from aicsimageio import exceptions
+from aicsimageio.readers.tiff_reader import TiffReader
-@pytest.mark.parametrize("name, expected", [
- ("s_1_t_1_c_1_z_1.tiff", True),
- ("s_1_t_10_c_3_z_1.tiff", True),
- ("example.png", False),
- ("s_1_t_1_c_1_z_1.ome.tiff", True)
-])
-def test_type_check(resources_dir, name, expected):
- tiff_image = resources_dir / name
- assert TiffReader.is_this_type(tiff_image) == expected
+@pytest.mark.parametrize(
+ "filename, "
+ "expected_shape, "
+ "expected_dims, "
+ "expected_dtype, "
+ "select_scene, "
+ "expected_chunksize, "
+ "expected_task_count",
+ [
+ # Expected task counts should be each non chunk dimension size multiplied againest each other * 2
+ (
+ "s_1_t_1_c_1_z_1.ome.tiff",
+ (325, 475),
+ "YX",
+ np.uint16,
+ 0,
+ (325, 475),
+ 2 # 2 = 2
+ ),
+ (
+ "s_1_t_1_c_1_z_1.tiff",
+ (325, 475),
+ "YX",
+ np.uint16,
+ 0,
+ (325, 475),
+ 2 # 2 = 2
+ ),
+ (
+ "s_1_t_1_c_10_z_1.ome.tiff",
+ (10, 1736, 1776),
+ "CYX",
+ np.uint16,
+ 0,
+ (1, 1736, 1776),
+ 20 # 2 = 2
+ ),
+ (
+ "s_1_t_10_c_3_z_1.tiff",
+ (10, 3, 325, 475),
+ "TCYX",
+ np.uint16,
+ 0,
+ (1, 1, 325, 475),
+ 60 # 10 * 3 * 2 = 60
+ ),
+ (
+ "s_3_t_1_c_3_z_5.ome.tiff",
+ (3, 5, 3, 325, 475),
+ "SZCYX",
+ np.uint16,
+ 0,
+ (1, 1, 1, 325, 475),
+ 90 # 3 * 5 * 3 * 2 = 90
+ ),
+ pytest.param(
+ "example.txt",
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ marks=pytest.mark.raises(exception=exceptions.UnsupportedFileFormatError)
+ )
+ ]
+)
+def test_tiff_reader(
+ resources_dir,
+ filename,
+ expected_shape,
+ expected_dims,
+ expected_dtype,
+ select_scene,
+ expected_chunksize,
+ expected_task_count
+):
+ # Get file
+ f = resources_dir / filename
+
+ # Read file
+ img = TiffReader(f, S=select_scene)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ assert img.dims == expected_dims
+ assert img.dtype() == expected_dtype
+ assert img.metadata
+ assert img.dask_data.shape == expected_shape
+ assert img.dask_data.chunksize == expected_chunksize
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check computed type is numpy array, computed shape is expected shape, and task count is expected
+ with Profiler() as prof:
+ assert isinstance(img.data, np.ndarray)
+ assert img.data.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+ # Check that there are no open file pointers after retrieval
+ assert str(f) not in [f.path for f in proc.open_files()]
-@pytest.mark.parametrize("name, shape, dims, metadata, dtype", [
- ("s_1_t_1_c_1_z_1.tiff", (325, 475), "YX", "ImageJ", np.uint16),
- ("s_1_t_10_c_3_z_1.tiff", (10, 3, 325, 475), "CZYX", "ImageJ", np.uint16),
- ("s_1_t_1_c_1_z_1.ome.tiff", (325, 475), "YX", "OME-XML", np.uint16)
+@pytest.mark.parametrize("expected_starting_dims, set_dims, expected_ending_dims", [
+ ("YX", "XY", "XY"),
+ pytest.param("YX", "ABCDE", None, marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError))
])
-def test_load(resources_dir, name, shape, dims, metadata, dtype):
- with TiffReader(resources_dir / name) as reader:
- data, actual_dims, actual_metadata = reader.load()
- assert data.shape == shape
- assert actual_dims == dims
- if metadata is None:
- assert actual_metadata == ""
- else:
- assert metadata in actual_metadata
- assert dtype == reader.dtype()
+def test_dims_setting(resources_dir, expected_starting_dims, set_dims, expected_ending_dims):
+ # Get file
+ f = resources_dir / "s_1_t_1_c_1_z_1.tiff"
+
+ # Read file
+ img = TiffReader(f)
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ assert img.dims == expected_starting_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check no tasks happen during dims setting
+ with Profiler() as prof:
+ img.dims = set_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check no tasks happen during dims getting
+ with Profiler() as prof:
+ assert img.dims == expected_ending_dims
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after retrieval
+ assert str(f) not in [f.path for f in proc.open_files()]
diff --git a/aicsimageio/tests/resources/variable_per_scene_dims.czi b/aicsimageio/tests/resources/variable_per_scene_dims.czi
new file mode 100755
index 000000000..e4a5c77eb
Binary files /dev/null and b/aicsimageio/tests/resources/variable_per_scene_dims.czi differ
diff --git a/aicsimageio/tests/test_aics_image.py b/aicsimageio/tests/test_aics_image.py
index 65a12f497..048f07de2 100644
--- a/aicsimageio/tests/test_aics_image.py
+++ b/aicsimageio/tests/test_aics_image.py
@@ -1,9 +1,16 @@
-from xml.etree import cElementTree as etree
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from unittest import mock
+
+import dask.array as da
import numpy as np
import pytest
+from dask.diagnostics import Profiler
+from lxml.etree import _Element
+from psutil import Process
-from aicsimageio import AICSImage, exceptions, imread, readers
+from aicsimageio import AICSImage, exceptions, imread, imread_dask, readers
from aicsimageio.vendor import omexml
# Example files
@@ -14,6 +21,9 @@
TIF_FILE = "s_1_t_1_c_1_z_1.tiff"
CZI_FILE = "s_1_t_1_c_1_z_1.czi"
OME_FILE = "s_1_t_1_c_1_z_1.ome.tiff"
+MED_TIF_FILE = "s_1_t_10_c_3_z_1.tiff"
+BIG_OME_FILE = "s_3_t_1_c_3_z_5.ome.tiff"
+BIG_CZI_FILE = "s_3_t_1_c_3_z_5.czi"
TXT_FILE = "example.txt"
@@ -40,21 +50,58 @@
],
)
def test_typing(filename, expected_reader, resources_dir):
- actual_reader = AICSImage.determine_reader(resources_dir / filename)
- assert actual_reader == expected_reader
+ # Get filepath
+ f = resources_dir / filename
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
-def test_file_passed_was_byte_string():
- with pytest.raises(exceptions.UnsupportedFileFormatError):
- AICSImage(b"not-a-valid-image-byte-array")
+ # Check basics
+ with Profiler() as prof:
+ actual_reader = AICSImage.determine_reader(f)
+ assert actual_reader == expected_reader
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
-@pytest.mark.parametrize(
- "arr", [np.zeros((2, 2, 2)), np.ones((2, 2, 2)), np.random.rand(2, 2, 2)]
-)
+
+def test_file_passed_was_directory(resources_dir):
+ # Get filepath
+ f = resources_dir
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ with pytest.raises(IsADirectoryError):
+ AICSImage(resources_dir)
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("arr", [
+ np.zeros((2, 2, 2)),
+ np.ones((2, 2, 2)),
+ np.random.rand(2, 2, 2),
+ da.zeros((2, 2, 2)),
+ da.ones((2, 2, 2)),
+ da.random.random((2, 2, 2))
+])
def test_support_for_ndarray(arr):
- actual_reader = AICSImage.determine_reader(arr)
- assert actual_reader == readers.NdArrayReader
+ # Check basics
+ with Profiler() as prof:
+ actual_reader = AICSImage.determine_reader(arr)
+ assert actual_reader == readers.ArrayLikeReader
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
@pytest.mark.parametrize(
@@ -63,11 +110,19 @@ def test_support_for_ndarray(arr):
(np.zeros((5, 4, 3)), (1, 1, 1, 5, 4, 3)),
(np.zeros((1, 2, 3, 4, 5, 6)), (1, 2, 3, 4, 5, 6)),
(np.random.rand(10), (1, 1, 1, 1, 1, 10)),
+ (da.zeros((5, 4, 3)), (1, 1, 1, 5, 4, 3)),
+ (da.zeros((1, 2, 3, 4, 5, 6)), (1, 2, 3, 4, 5, 6)),
+ (da.random.random((10)), (1, 1, 1, 1, 1, 10)),
],
)
-def test_default_dims(data, expected):
- img = AICSImage(data=data)
- assert img.data.shape == expected
+def test_default_shape_expansion(data, expected):
+ # Check basics
+ with Profiler() as prof:
+ img = AICSImage(data=data)
+ assert img.dask_data.shape == expected
+ assert img.shape == expected
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
@pytest.mark.parametrize(
@@ -80,20 +135,34 @@ def test_default_dims(data, expected):
np.zeros((2, 2, 2)),
"ABI",
None,
- marks=pytest.mark.raises(exception=TypeError),
+ marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError),
+ ),
+ (da.zeros((5, 4, 3)), "SYX", (5, 1, 1, 1, 4, 3)),
+ (da.zeros((1, 2, 3, 4, 5)), "STCYX", (1, 2, 3, 1, 4, 5)),
+ (da.zeros((10, 20)), "XY", (1, 1, 1, 1, 20, 10)),
+ pytest.param(
+ da.zeros((2, 2, 2)),
+ "ABI",
+ None,
+ marks=pytest.mark.raises(exception=exceptions.InvalidDimensionOrderingError),
),
],
)
def test_known_dims(data, dims, expected_shape):
- img = AICSImage(data, known_dims=dims)
- assert img.data.shape == expected_shape
- assert img.size_x == expected_shape[5]
- assert img.size_y == expected_shape[4]
- assert img.size_z == expected_shape[3]
- assert img.size_c == expected_shape[2]
- assert img.size_t == expected_shape[1]
- assert img.size_s == expected_shape[0]
- assert img.size(dims) == data.shape
+ # Check basics
+ with Profiler() as prof:
+ img = AICSImage(data, known_dims=dims)
+ assert img.data.shape == expected_shape
+ assert img.size_x == expected_shape[5]
+ assert img.size_y == expected_shape[4]
+ assert img.size_z == expected_shape[3]
+ assert img.size_c == expected_shape[2]
+ assert img.size_t == expected_shape[1]
+ assert img.size_s == expected_shape[0]
+ assert img.size(dims) == data.shape
+
+ # Due to reshape and transpose there will be 2 tasks in the graph
+ assert len(prof.results) == 2
@pytest.mark.parametrize(
@@ -106,69 +175,53 @@ def test_known_dims(data, dims, expected_shape):
],
)
def test_force_dims(data_shape, dims, expected):
- img = AICSImage(data=np.zeros(data_shape))
- img._reader.dims = dims
- assert img.data.shape == expected
- assert data_shape == img.get_image_data(out_orientation=dims).shape
- assert img.size_x == expected[5]
- assert img.size_y == expected[4]
- assert img.size_z == expected[3]
- assert img.size_c == expected[2]
- assert img.size_t == expected[1]
- assert img.size_s == expected[0]
- assert img.size(dims) == data_shape
+ # Check basics
+ with Profiler() as prof:
+ img = AICSImage(data=da.zeros(data_shape))
+ img._reader._dims = dims
+ assert img.data.shape == expected
+ assert data_shape == img.get_image_data(out_orientation=dims).shape
+ assert img.size_x == expected[5]
+ assert img.size_y == expected[4]
+ assert img.size_z == expected[3]
+ assert img.size_c == expected[2]
+ assert img.size_t == expected[1]
+ assert img.size_s == expected[0]
+ assert img.size(dims) == data_shape
-
-@pytest.mark.parametrize(
- "filepath",
- [
- OME_FILE,
- pytest.param(
- "fakeimage.ome.tif", marks=pytest.mark.raises(exception=FileNotFoundError)
- ),
- ],
-)
-def test_file_exceptions(resources_dir, filepath):
- f = resources_dir / filepath
- AICSImage(f)
-
-
-def test_file_passed_was_directory(resources_dir):
- with pytest.raises(IsADirectoryError):
- AICSImage(resources_dir)
+ # Two operations are happening
+ # First, img.data is called and so two tasks of reshape and transpose are ran
+ # Then get_image_data is ran and two more reshape and transpose are ran
+ assert len(prof.results) == 4
@pytest.mark.parametrize(
"filename, expected_metadata_type",
[
- (PNG_FILE, (str, type(None))),
- (TIF_FILE, (str, type(None))),
- (OME_FILE, (str, omexml.OMEXML)),
- (CZI_FILE, (str, etree.Element)),
+ (PNG_FILE, dict),
+ (TIF_FILE, str),
+ (OME_FILE, omexml.OMEXML),
+ (CZI_FILE, _Element),
],
)
def test_metadata(resources_dir, filename, expected_metadata_type):
- img = AICSImage(resources_dir / filename)
- assert isinstance(img.metadata, expected_metadata_type)
+ # Get filepath
+ f = resources_dir / filename
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
-@pytest.mark.parametrize(
- "filename, expected_reader",
- [
- (PNG_FILE, readers.DefaultReader),
- (TIF_FILE, readers.TiffReader),
- (OME_FILE, readers.OmeTiffReader),
- (CZI_FILE, readers.CziReader),
- pytest.param(
- "not/a/file.czi",
- None,
- marks=pytest.mark.raises(exception=FileNotFoundError),
- ),
- ],
-)
-def test_reader(resources_dir, filename, expected_reader):
- img = AICSImage(resources_dir / filename)
- assert isinstance(img.reader, expected_reader)
+ # Check basics
+ with Profiler() as prof:
+ img = AICSImage(f)
+ assert isinstance(img.metadata, expected_metadata_type)
+
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
@pytest.mark.parametrize(
@@ -181,8 +234,71 @@ def test_reader(resources_dir, filename, expected_reader):
],
)
def test_imread(resources_dir, filename, expected_shape):
- img = imread(resources_dir / filename)
- assert img.shape == expected_shape
+ # Get filepath
+ f = resources_dir / filename
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ img = imread(f)
+ assert img.shape == expected_shape
+
+ # Reshape and transpose are required so there should be two tasks in the graph
+ assert len(prof.results) == 2
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("filename, expected_shape, expected_task_count", [
+ # Because we are directly requesting the data, the transpose and reshape calls get reduced
+ (MED_TIF_FILE, (1, 10, 3, 1, 325, 475), 60),
+ (BIG_OME_FILE, (3, 1, 3, 5, 325, 475), 90),
+ (BIG_CZI_FILE, (3, 1, 3, 5, 325, 475), 18)
+])
+def test_large_imread(resources_dir, filename, expected_shape, expected_task_count):
+ # Get filepath
+ f = resources_dir / filename
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ img = imread(f)
+ assert img.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("filename, expected_shape, expected_task_count", [
+ # Because we are directly returning the dask array nothing has been computed
+ (MED_TIF_FILE, (1, 10, 3, 1, 325, 475), 0),
+ (BIG_OME_FILE, (3, 1, 3, 5, 325, 475), 0),
+ (BIG_CZI_FILE, (3, 1, 3, 5, 325, 475), 0)
+])
+def test_large_imread_dask(resources_dir, filename, expected_shape, expected_task_count):
+ # Get filepath
+ f = resources_dir / filename
+
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+ # Check basics
+ with Profiler() as prof:
+ img = imread_dask(f)
+ assert img.shape == expected_shape
+ assert len(prof.results) == expected_task_count
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
@pytest.mark.parametrize(
@@ -195,24 +311,134 @@ def test_imread(resources_dir, filename, expected_shape):
],
)
def test_channel_names(resources_dir, filename, expected_channel_names):
- img = AICSImage(resources_dir / filename)
- assert img.get_channel_names() == expected_channel_names
+ # Get filepath
+ f = resources_dir / filename
+ # Check that there are no open file pointers after init
+ proc = Process()
+ assert str(f) not in [f.path for f in proc.open_files()]
-@pytest.mark.parametrize(
- "filename",
- [
- PNG_FILE,
- TIF_FILE,
- CZI_FILE,
- OME_FILE,
- ],
-)
-def test_aicsimage_close(resources_dir, filename):
- img = AICSImage(resources_dir / filename)
- assert img.reader._bytes.closed is False
- img.close()
-
- with AICSImage(resources_dir / filename) as img:
- img.metadata
- assert img.reader._bytes.closed is True
+ # Check basics
+ with Profiler() as prof:
+ img = AICSImage(f)
+ assert img.get_channel_names() == expected_channel_names
+
+ # Check that basic details don't require task computation
+ assert len(prof.results) == 0
+
+ # Check that there are no open file pointers after basics
+ assert str(f) not in [f.path for f in proc.open_files()]
+
+
+@pytest.mark.parametrize("data, rgb, expected_data, expected_visible, expected_ndim, expected_axis_labels", [
+ (
+ # C Z Y X
+ np.ones((3, 2, 2, 2)),
+ False,
+ da.ones((3, 2, 2, 2)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # C Z Y X
+ da.ones((3, 2, 2, 2)),
+ False,
+ da.ones((3, 2, 2, 2)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # C Z Y X
+ np.ones((3, 2, 2, 2)),
+ True,
+ da.ones((2, 2, 2, 3)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # C Z Y X
+ da.ones((3, 2, 2, 2)),
+ True,
+ da.ones((2, 2, 2, 3)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # S T C Z Y X
+ np.ones((1, 1, 3, 2, 2, 2)),
+ False,
+ da.ones((3, 2, 2, 2)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # S T C Z Y X
+ da.ones((1, 1, 3, 2, 2, 2)),
+ False,
+ da.ones((3, 2, 2, 2)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # S T C Z Y X
+ np.ones((1, 1, 3, 2, 2, 2)),
+ True,
+ da.ones((2, 2, 2, 3)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # S T C Z Y X
+ da.ones((1, 1, 3, 2, 2, 2)),
+ True,
+ da.ones((2, 2, 2, 3)),
+ True,
+ 3,
+ "ZYX"
+ ),
+ (
+ # S T C Z Y X
+ np.ones((3, 20, 5, 2, 2, 2)),
+ False,
+ da.ones((3, 20, 5, 2, 2, 2)),
+ False,
+ 3,
+ "STZYX"
+ ),
+ (
+ # S T C Z Y X
+ da.ones((3, 20, 5, 2, 2, 2)),
+ False,
+ da.ones((3, 20, 5, 2, 2, 2)),
+ False,
+ 3,
+ "STZYX"
+ ),
+])
+def test_view_napari(data, rgb, expected_data, expected_visible, expected_ndim, expected_axis_labels):
+ # Init image
+ img = AICSImage(data)
+
+ # Mock napari view
+ with mock.patch("napari.gui_qt"):
+ with mock.patch("napari.view_image") as mocked_napari:
+ img.view_napari(rgb)
+
+ # Check array equal
+ args = mocked_napari.call_args[0]
+ assert args[0].shape == expected_data.shape
+
+ # Check extra call kwargs
+ call_kwargs = mocked_napari.call_args[1]
+ assert not call_kwargs["is_pyramid"]
+ assert call_kwargs["ndisplay"] == expected_ndim
+ assert call_kwargs["axis_labels"] == expected_axis_labels
+ if not rgb:
+ assert call_kwargs["visible"] == expected_visible
diff --git a/aicsimageio/tests/test_dask_utils.py b/aicsimageio/tests/test_dask_utils.py
new file mode 100644
index 000000000..a08b7ead4
--- /dev/null
+++ b/aicsimageio/tests/test_dask_utils.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from unittest import mock
+
+import pytest
+
+from aicsimageio import AICSImage
+from aicsimageio.readers import CziReader
+
+
+@pytest.mark.parametrize("address, nworkers", [
+ (None, None),
+ (None, 2),
+ ("tcp://some-spawned-cluster", None),
+])
+def test_aicsimage_context_manager(resources_dir, address, nworkers):
+ # Patch the spawn function
+ with mock.patch("aicsimageio.dask_utils.spawn_cluster_and_client") as mocked_spawner:
+ return "a", "b"
+
+ # Patch the shutdown function
+ with mock.patch("aicsimageio.dask_utils.shutdown_cluster_and_client") as mocked_shutdown:
+ return "a", "b"
+
+ # Load the image in a context manager that spawn and closes a cluster and client
+ with AICSImage(resources_dir / "s_3_t_1_c_3_z_5.czi", address=address, nworkers=nworkers):
+ assert mocked_spawner.call_args[0][0] == address
+ assert mocked_spawner.call_arg[1]["nworkers"] == nworkers
+
+ # Check that the cluster and client were scheduled to shutdown
+ assert mocked_shutdown.called
+
+
+@pytest.mark.parametrize("address, nworkers", [
+ (None, None),
+ (None, 2),
+ ("tcp://some-spawned-cluster", None),
+])
+def test_reader_context_manager(resources_dir, address, nworkers):
+ # Patch the spawn function
+ with mock.patch("aicsimageio.dask_utils.spawn_cluster_and_client") as mocked_spawner:
+ return "a", "b"
+
+ # Patch the shutdown function
+ with mock.patch("aicsimageio.dask_utils.shutdown_cluster_and_client") as mocked_shutdown:
+ return "a", "b"
+
+ # Load the image in a context manager that spawn and closes a cluster and client
+ with CziReader(resources_dir / "s_3_t_1_c_3_z_5.czi", address=address, nworkers=nworkers):
+ assert mocked_spawner.call_args[0][0] == address
+ assert mocked_spawner.call_arg[1]["nworkers"] == nworkers
+
+ # Check that the cluster and client were scheduled to shutdown
+ assert mocked_shutdown.called
diff --git a/aicsimageio/tests/test_transforms.py b/aicsimageio/tests/test_transforms.py
index 796b8cc72..1f608934b 100644
--- a/aicsimageio/tests/test_transforms.py
+++ b/aicsimageio/tests/test_transforms.py
@@ -1,46 +1,173 @@
-import pytest
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import dask.array as da
import numpy as np
-from aicsimageio.transforms import transpose_to_dims, reshape_data
+import pytest
+
from aicsimageio.exceptions import ConflictingArgumentsError
+from aicsimageio.transforms import reshape_data, transpose_to_dims
-# test _reshape_data which is called when data is accessed
-@pytest.mark.parametrize("data, given_dims, return_dims, other_args, expected", [
+@pytest.mark.parametrize("data, given_dims, return_dims, other_args, expected_shape", [
(np.zeros((10, 1, 5, 6, 200, 400)), "STCZYX", "CSZYX", {}, (5, 10, 6, 200, 400)),
+ (da.zeros((10, 1, 5, 6, 200, 400)), "STCZYX", "CSZYX", {}, (5, 10, 6, 200, 400)),
(np.zeros((6, 200, 400)), "ZYX", "STCZYX", {}, (1, 1, 1, 6, 200, 400)),
+ (da.zeros((6, 200, 400)), "ZYX", "STCZYX", {}, (1, 1, 1, 6, 200, 400)),
(np.zeros((6, 200, 400)), "ZYX", "ZCYSXT", {}, (6, 1, 200, 1, 400, 1)),
- pytest.param(np.zeros((6, 200, 400)), 'ZYX', 'TYXC', {'Z': 7}, 5,
- marks=pytest.mark.raises(exception=IndexError)),
- pytest.param(np.zeros((6, 200, 400)), 'ZYX', 'TYXCZ', {'Z': 7}, 5,
- marks=pytest.mark.raises(exception=ConflictingArgumentsError)),
- pytest.param(np.zeros((6, 200, 400)), 'ZYX', 'TYXCZX', {'Z': 7}, 5,
- marks=pytest.mark.raises(exception=ConflictingArgumentsError)),
+ (da.zeros((6, 200, 400)), "ZYX", "ZCYSXT", {}, (6, 1, 200, 1, 400, 1)),
+ (np.zeros((2, 2, 2)), "ABI", "ZCYSXT", {}, (1, 1, 1, 1, 1, 1)),
+ (da.zeros((2, 2, 2)), "ABI", "ZCYSXT", {}, (1, 1, 1, 1, 1, 1)),
+ pytest.param(
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXC",
+ {"Z": 7},
+ None,
+ marks=pytest.mark.raises(exception=IndexError)
+ ),
+ pytest.param(
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXC",
+ {"Z": 7},
+ None,
+ marks=pytest.mark.raises(exception=IndexError)
+ ),
+ pytest.param(
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXCZ",
+ {"Z": 7},
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
+ pytest.param(
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXCZ",
+ {"Z": 7},
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
+ pytest.param(
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXCZX",
+ {"Z": 7},
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
+ pytest.param(
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXCZX",
+ {"Z": 7},
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
])
-def test_reshape_data_shape(data, given_dims, return_dims, other_args, expected):
- # the other_args are being used to pass slice specific information by expanding with the ** operator
- ans = reshape_data(data=data, given_dims=given_dims, return_dims=return_dims, **other_args)
- assert ans.shape == expected
+def test_reshape_data_shape(data, given_dims, return_dims, other_args, expected_shape):
+ actual = reshape_data(data=data, given_dims=given_dims, return_dims=return_dims, **other_args)
+ assert actual.shape == expected_shape
+
+ # Check that the output data is the same type as the input
+ assert type(actual) == type(data)
@pytest.mark.parametrize("data, given_dims, return_dims, idx_in, idx_out", [
- (np.random.rand(10, 1, 5, 6, 200, 400), "STCZYX", "CSZYX", (5, 0, 3, 3, ...), (3, 5, 3, ...)),
- (np.zeros((6, 200, 400)), "ZYX", "STCZYX", (..., 100, 200), (0, 0, 0, ..., 100, 200)),
- (np.zeros((6, 200, 400)), "ZYX", "ZCYSXT", (3, 100, ...), (3, 0, 100, 0, ..., 0)),
+ (
+ np.random.rand(10, 1, 5, 6, 200, 400),
+ "STCZYX",
+ "CSZYX",
+ (5, 0, 3, 3, ...),
+ (3, 5, 3, ...)
+ ),
+ (
+ da.random.random((10, 1, 5, 6, 200, 400)),
+ "STCZYX",
+ "CSZYX",
+ (5, 0, 3, 3, ...),
+ (3, 5, 3, ...)
+ ),
+ (
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "STCZYX",
+ (..., 100, 200),
+ (0, 0, 0, ..., 100, 200)
+ ),
+ (
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "STCZYX",
+ (..., 100, 200),
+ (0, 0, 0, ..., 100, 200)
+ ),
+ (
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "ZCYSXT",
+ (3, 100, ...),
+ (3, 0, 100, 0, ..., 0)
+ ),
+ (
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "ZCYSXT",
+ (3, 100, ...),
+ (3, 0, 100, 0, ..., 0)
+ ),
])
def test_reshape_data_values(data, given_dims, return_dims, idx_in, idx_out):
slice_in = data[idx_in]
- ans = reshape_data(data=data, given_dims=given_dims, return_dims=return_dims)
- np.testing.assert_array_equal(slice_in, ans[idx_out])
+ actual = reshape_data(data=data, given_dims=given_dims, return_dims=return_dims)
+ if isinstance(data, da.core.Array):
+ slice_in = slice_in.compute()
+ actual = actual.compute()
+ np.testing.assert_array_equal(slice_in, actual[idx_out])
+
+ # Check that the output data is the same type as the input
+ assert type(actual) == type(slice_in)
-@pytest.mark.parametrize("data, given_dims, return_dims, expected", [
+@pytest.mark.parametrize("data, given_dims, return_dims, expected_shape", [
(np.zeros((1, 2, 3, 4, 5, 6)), "STCZYX", "XYZCTS", (6, 5, 4, 3, 2, 1)),
+ (da.zeros((1, 2, 3, 4, 5, 6)), "STCZYX", "XYZCTS", (6, 5, 4, 3, 2, 1)),
(np.zeros((1, 2, 3)), "ZYX", "ZXY", (1, 3, 2)),
- pytest.param(np.zeros((6, 200, 400)), 'ZYX', 'TYXC', 5,
- marks=pytest.mark.raises(exception=ConflictingArgumentsError)),
- pytest.param(np.zeros((6, 200, 400)), 'ZYX', 'TYXCZ', 5,
- marks=pytest.mark.raises(exception=ConflictingArgumentsError)),
+ (da.zeros((1, 2, 3)), "ZYX", "ZXY", (1, 3, 2)),
+ pytest.param(
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXC",
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
+ pytest.param(
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXC",
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
+ pytest.param(
+ np.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXCZ",
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
+ pytest.param(
+ da.zeros((6, 200, 400)),
+ "ZYX",
+ "TYXCZ",
+ None,
+ marks=pytest.mark.raises(exception=ConflictingArgumentsError)
+ ),
])
-def test_transpose_to_dims(data, given_dims, return_dims, expected):
- data = transpose_to_dims(data=data, given_dims=given_dims, return_dims=return_dims)
- assert data.shape == expected
+def test_transpose_to_dims(data, given_dims, return_dims, expected_shape):
+ actual = transpose_to_dims(data=data, given_dims=given_dims, return_dims=return_dims)
+ assert actual.shape == expected_shape
+
+ # Check that the output data is the same type as the input
+ assert type(actual) == type(data)
diff --git a/aicsimageio/tests/writers/test_ome_tiff_writer.py b/aicsimageio/tests/writers/test_ome_tiff_writer.py
index 4b7b44de8..94c4e691f 100644
--- a/aicsimageio/tests/writers/test_ome_tiff_writer.py
+++ b/aicsimageio/tests/writers/test_ome_tiff_writer.py
@@ -1,13 +1,14 @@
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
import os
-import pytest
import numpy as np
+import pytest
+from aicsimageio.exceptions import InvalidDimensionOrderingError
from aicsimageio.readers.ome_tiff_reader import OmeTiffReader
from aicsimageio.writers import OmeTiffWriter
-from aicsimageio.exceptions import InvalidDimensionOrderingError
filename = "ometif_test_output.ome.tif"
@@ -18,12 +19,10 @@ def test_writerShapeComparison(resources_dir):
"""
Test to check that OmeTiffWriter saves arrays that are reflexive with OmeTiffReader
"""
- writer = OmeTiffWriter(resources_dir / filename, overwrite_file=True)
- writer.save(image)
- writer.close()
+ with OmeTiffWriter(resources_dir / filename, overwrite_file=True) as writer:
+ writer.save(image)
- with OmeTiffReader(resources_dir / filename) as test_output_reader:
- output = test_output_reader.data
+ output = OmeTiffReader(resources_dir / filename).data
assert output.shape == image.shape[1:]
@@ -34,8 +33,8 @@ def test_loadAssertionError(resources_dir):
"""
image_to_save = np.ones((1, 2, 3, 4, 5, 6))
with pytest.raises(Exception):
- writer = OmeTiffWriter(resources_dir / filename, overwrite_file=True)
- writer.save(image_to_save)
+ with OmeTiffWriter(resources_dir / filename, overwrite_file=True) as writer:
+ writer.save(image_to_save)
def test_overwriteFile(resources_dir):
@@ -121,17 +120,16 @@ def test_big_tiff():
def test_dimensionOrder(
resources_dir, dims, expected_t, expected_c, expected_z, expected_y, expected_x
):
- writer = OmeTiffWriter(resources_dir / filename, overwrite_file=True)
- writer.save(image, dimension_order=dims)
- writer.close()
-
- with OmeTiffReader(resources_dir / filename) as test_output_reader:
- output = test_output_reader.data
- t = test_output_reader.size_t()
- c = test_output_reader.size_c()
- z = test_output_reader.size_z()
- y = test_output_reader.size_y()
- x = test_output_reader.size_x()
+ with OmeTiffWriter(resources_dir / filename, overwrite_file=True) as writer:
+ writer.save(image, dimension_order=dims)
+
+ reader = OmeTiffReader(resources_dir / filename)
+ output = reader.data
+ t = reader.size_t()
+ c = reader.size_c()
+ z = reader.size_z()
+ y = reader.size_y()
+ x = reader.size_x()
os.remove(resources_dir / filename)
diff --git a/aicsimageio/tests/writers/test_png_writer.py b/aicsimageio/tests/writers/test_png_writer.py
index 00d22ef2a..d95dda35a 100644
--- a/aicsimageio/tests/writers/test_png_writer.py
+++ b/aicsimageio/tests/writers/test_png_writer.py
@@ -38,7 +38,6 @@ def test_pngSaveComparison(self):
self.writer.save(self.image.astype('uint8'))
reader = DefaultReader(self.file)
output_image = reader.data.T
- reader.close()
self.assertTrue(np.array_equal(self.image, output_image))
"""
@@ -49,14 +48,12 @@ def test_pngSaveImageComparison(self):
self.writer.save_slice(self.image.astype('uint8'), z=1, c=2, t=3)
reader = DefaultReader(self.file)
output_image = reader.data.T
- reader.close()
self.assertTrue(np.array_equal(self.image, output_image))
"""
Test to check if save() can overwrite a file
"""
def test_overwriteFile(self):
- print(self.file)
with PngWriter(self.file, overwrite_file=True) as writer:
writer.save(self.image.astype('uint8'))
@@ -91,9 +88,9 @@ def test_twoDimensionalImages(self):
image[1, 0] = 0
image[1, 1] = 255
self.writer.save(image)
- with DefaultReader(self.file) as reader:
- loaded_image = reader.data.T
- self.assertTrue(np.array_equal(image, loaded_image))
+ reader = DefaultReader(self.file)
+ loaded_image = reader.data.T
+ self.assertTrue(np.array_equal(image, loaded_image))
"""
Test saves an image with a single xy plane, but gives one channel
@@ -106,13 +103,13 @@ def test_threeDimensionalImages(self):
image[0, 1, 0] = 0
image[0, 1, 1] = 255
self.writer.save(image)
- with DefaultReader(self.file) as reader:
- all_channels = reader.data.T
- channel_r = all_channels[0, :, :]
- channel_g = all_channels[1, :, :]
- channel_b = all_channels[2, :, :]
- self.assertTrue(np.array_equal(channel_r, channel_g) and np.array_equal(channel_g, channel_b)
- and np.array_equal(channel_r, image[0, :, :]))
+ reader = DefaultReader(self.file)
+ all_channels = reader.data.T
+ channel_r = all_channels[0, :, :]
+ channel_g = all_channels[1, :, :]
+ channel_b = all_channels[2, :, :]
+ self.assertTrue(np.array_equal(channel_r, channel_g) and np.array_equal(channel_g, channel_b)
+ and np.array_equal(channel_r, image[0, :, :]))
"""
Test attempts to save an image with zcyx dims
diff --git a/aicsimageio/transforms.py b/aicsimageio/transforms.py
index 271b646a7..8161766af 100644
--- a/aicsimageio/transforms.py
+++ b/aicsimageio/transforms.py
@@ -1,101 +1,179 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
import logging
-import warnings
from collections import Counter
+import dask.array as da
import numpy as np
+from . import types
from .exceptions import ConflictingArgumentsError
+###############################################################################
+
log = logging.getLogger(__name__)
+###############################################################################
+
+
+def _array_split(operator, ary, indices_or_sections, axis=0):
+ """
+ A reimplementation of array split that doesn't cast to numpy and returns a list of
+ whichever object type it is currently operating on.
+
+ https://github.com/numpy/numpy/blob/v1.17.0/numpy/lib/shape_base.py#L723
+ """
+ try:
+ Ntotal = ary.shape[axis]
+ except AttributeError:
+ Ntotal = len(ary)
+ try:
+ # handle array case.
+ Nsections = len(indices_or_sections) + 1
+ div_points = [0] + list(indices_or_sections) + [Ntotal]
+ except TypeError:
+ # indices_or_sections is a scalar, not an array.
+ Nsections = int(indices_or_sections)
+ if Nsections <= 0:
+ raise ValueError("number sections must be larger than 0.")
+ Neach_section, extras = divmod(Ntotal, Nsections)
+ section_sizes = ([0] + extras * [Neach_section+1] + (Nsections-extras) * [Neach_section])
+ div_points = np.array(section_sizes, dtype=ary.dtype).cumsum(axis=None)
+
+ sub_arys = []
+ sary = operator.swapaxes(ary, axis, 0)
+ for i in range(Nsections):
+ st = div_points[i]
+ end = div_points[i + 1]
+
+ # Make sure start and end are integers
+ st = int(st)
+ end = int(end)
+
+ sub_arys.append(operator.swapaxes(sary[st:end], axis, 0))
+
+ return sub_arys
+
-def reshape_data(data: np.ndarray, given_dims: str, return_dims: str, copy: bool = False, **kwargs) -> np.ndarray:
+def _split(operator, ary, indices_or_sections, axis=0):
+ """
+ A reimplementation of split that doesn't cast to numpy and returns a list of
+ whichever object type it is currently operating on.
+
+ https://github.com/numpy/numpy/blob/v1.17.0/numpy/lib/shape_base.py#L782
+ """
+ try:
+ len(indices_or_sections)
+ except TypeError:
+ sections = indices_or_sections
+ N = ary.shape[axis]
+ if N % sections:
+ raise ValueError("array split does not result in an equal division")
+ return _array_split(operator, ary, indices_or_sections, axis)
+
+
+def reshape_data(
+ data: types.ArrayLike,
+ given_dims: str,
+ return_dims: str,
+ **kwargs
+) -> types.ArrayLike:
"""
Reshape the data into return_dims, pad missing dimensions, and prune extra dimensions.
Warns the user to use the base reader if the depth of the Dimension being removed is not 1.
Parameters
----------
- data: a numpy.ndarray of arbitrary shape but with the dimensions specified in given_dims
- given_dims: the dimension ordering of data, "CZYX", "VBTCXZY" etc
- return_dims: the dimension ordering of the return data
- copy: True or False, if true copy the data object if false modify the object/view into the object
+ data: types.ArrayLike
+ Either a dask array or numpy.ndarray of arbitrary shape but with the dimensions specified in given_dims
+ given_dims: str
+ The dimension ordering of data, "CZYX", "VBTCXZY" etc
+ return_dims: str
+ The dimension ordering of the return data
kwargs:
C=1 => desired specific channel, if C in the input data has depth 3 then C=1 returns the 2nd slice (0 indexed)
Z=10 => desired specific channel, if Z in the input data has depth 20 then Z=10 returns the 11th slice
+
Returns
-------
- a numpy.ndarray in return_dims order, if return_dims=DEFAULT_DIMS then the return would have order "STCZYX"
+ data: types.ArrayLike
+ An array in return_dims order, if return_dims=DEFAULT_DIMS then the return would have order "STCZYX"
"""
- # copy the data object if copy=True is in kwargs
- data = data.copy() if copy else data
- # check for conflicts like return_dims='TCZYX' and fixed channels 'C=1'
+ # Get operator
+ if isinstance(data, da.core.Array):
+ operator = da
+ else:
+ operator = np
+
+ # Check for conflicts like return_dims='TCZYX' and fixed channels 'C=1'
for dim in return_dims:
if kwargs.get(dim) is not None:
msg = f"Argument return_dims={return_dims} and argument {dim}={kwargs.get(dim)} conflict. Check usage."
raise ConflictingArgumentsError(msg)
- # add each dimension not included in original data
+ # Add each dimension not included in original data
new_dims = given_dims
excluded_dims = "".join(set(return_dims) - set(given_dims))
for dim in excluded_dims:
- data = np.expand_dims(data, axis=0)
+ # Dask doesn't have an explicit expand dims so we simply reshape with an extra dim
+ data = operator.reshape(data, (1, *data.shape))
new_dims = dim + new_dims # add the missing Dimension to the front
- # if given dims contains a Dimension not in DEFAULT_DIMS and its depth is 1 remove it
- # if it's larger than 1 give a warning and suggest interfacing with the Reader object
+ # If given dims contains a Dimension not in return dims and its depth is 1 remove it
+ # If it's larger than 1 give a warning and suggest interfacing with the Reader object
extra_dims = "".join(set(given_dims) - set(return_dims))
for dim in extra_dims:
index = new_dims.find(dim)
if data.shape[index] > 1:
index_depth = kwargs.get(dim)
if index_depth is None:
- msg = (f'Data has dimension {dim} with depth {data.shape[index]}, assuming {dim}=0 is '
- f'the desired value, if not the case specify {dim}=x where '
- f'x is an integer in [0, {data.shape[index]}).')
- warnings.warn(msg=msg, category=UserWarning, stacklevel=2)
+ log.warning(
+ f"Data has dimension {dim} with depth {data.shape[index]}, assuming {dim}=0 is "
+ f"the desired value, if not the case specify {dim}=x where "
+ f"x is an integer in [0, {data.shape[index]})."
+ )
index_depth = 0
if index_depth >= data.shape[index]:
- raise IndexError(f'Dimension specified with {dim}={index_depth} '
- f'but Dimension shape is {data.shape[index]}.')
- planes = np.split(data, data.shape[index], axis=index) # split dim into list of arrays
+ raise IndexError(f"Dimension specified with {dim}={index_depth} "
+ f"but Dimension shape is {data.shape[index]}.")
+ planes = _split(operator, data, data.shape[index], axis=index) # split dim into list of arrays
data = planes[index_depth] # take the specified value of the dim
- data = np.squeeze(data, axis=index) # remove the dim from ndarray
+ data = operator.squeeze(data, axis=index) # remove the dim from ndarray
new_dims = new_dims[0:index] + new_dims[index + 1:] # clip out the Dimension from new_dims
- # any extra dimensions have been removed, only a problem if the depth is > 1
+ # Any extra dimensions have been removed, only a problem if the depth is > 1
return transpose_to_dims(data, given_dims=new_dims, return_dims=return_dims) # don't pass kwargs or 2 copies
-def transpose_to_dims(data: np.ndarray, given_dims: str, return_dims: str, copy: bool = False) -> np.ndarray:
+def transpose_to_dims(
+ data: types.ArrayLike,
+ given_dims: str,
+ return_dims: str,
+) -> types.ArrayLike:
"""
This shuffles the data dimensions from given_dims to return_dims. Each dimension must be present in
given_dims must be used in return_dims
- Parameters
- ----------
- data: the input data with dimensions given_dims
- given_dims: the dimensions of data
- return_dims: the reordered set of given_dims to return
- copy: if True copy the data object passed in if false modify the numpy.ndarray
+ data: types.ArrayLike
+ Either a dask array or numpy.ndarray of arbitrary shape but with the dimensions specified in given_dims
+ given_dims: str
+ The dimension ordering of data, "CZYX", "VBTCXZY" etc
+ return_dims: str
+ The dimension ordering of the return data
Returns
-------
- a numpy.ndarray with known_dims
-
+ data: types.ArrayLike
+ An array in return_dims order, if return_dims=DEFAULT_DIMS then the return would have order "STCZYX"
"""
- # copy the data object if copy=True is in kwargs
- data = data.copy() if copy else data
-
# Use a counter to track that the contents are composed of the same letters and that no letter is repeated
if Counter(given_dims) != Counter(return_dims) or max(Counter(given_dims).values()) > 1:
raise ConflictingArgumentsError(f"given_dims={given_dims} and return_dims={return_dims} are incompatible.")
- # resort the data into return_dims order
+ # Resort the data into return_dims order
match_map = {dim: given_dims.find(dim) for dim in given_dims}
transposer = []
for dim in return_dims:
- if match_map[dim] == -1:
- raise ConflictingArgumentsError(f'Dimension {dim} requested but not present in given_dims={given_dims}.')
transposer.append(match_map[dim])
data = data.transpose(transposer)
return data
diff --git a/aicsimageio/types.py b/aicsimageio/types.py
index 0a44e5ed3..d069f3c01 100644
--- a/aicsimageio/types.py
+++ b/aicsimageio/types.py
@@ -7,8 +7,9 @@
from io import BufferedIOBase
from pathlib import Path
-from typing import Union, NamedTuple, Any
+from typing import Any, NamedTuple, Union
+import dask.array as da
import numpy as np
# Imaging Data Types
@@ -18,7 +19,8 @@
PathLike = Union[str, Path]
BufferLike = Union[bytes, BufferedIOBase]
FileLike = Union[PathLike, BufferLike]
-ImageLike = Union[FileLike, np.ndarray]
+ArrayLike = Union[np.ndarray, da.core.Array]
+ImageLike = Union[FileLike, ArrayLike]
class LoadResults(NamedTuple):
diff --git a/aicsimageio/vendor/czifile.py b/aicsimageio/vendor/czifile.py
deleted file mode 100644
index defd25d3e..000000000
--- a/aicsimageio/vendor/czifile.py
+++ /dev/null
@@ -1,1327 +0,0 @@
-# -*- coding: utf-8 -*-
-# czifile.py
-
-# Copyright (c) 2013-2017, Christoph Gohlke
-# Copyright (c) 2013-2017, The Regents of the University of California
-# Produced at the Laboratory for Fluorescence Dynamics.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-# * Neither the name of the copyright holders nor the names of any
-# contributors may be used to endorse or promote products derived
-# from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-"""Read image and metadata from Carl Zeiss(r) ZISRAW (CZI) files.
-
-CZI is the native image file format of the ZEN(r) software by Carl Zeiss
-Microscopy GmbH. It stores multidimensional images and metadata from
-microscopy experiments.
-
-:Author:
- `Christoph Gohlke `_
-
-:Organization:
- Laboratory for Fluorescence Dynamics, University of California, Irvine
-
-:Version: 2017.09.12
-
-Requirements
-------------
-* `CPython 3.6 64-bit `_
-* `Numpy 1.13 `_
-* `Scipy 0.19 `_
-* `Tifffile.py 2017.09.12 `_
-* `Czifle.pyx 2017.07.20 `_
- (for decoding JpegXrFile and JpgFile images)
-
-Revisions
----------
-2017.09.12
- Require tifffile.py 2017.09.12
-2017.07.21
- Use multi-threading in CziFile.asarray to decode and copy segment data.
- Always convert BGR to RGB. Remove bgr2rgb options.
- Decode JpegXR directly from byte arrays.
-2017.07.13
- Add function to convert CZI file to memory-mappable TIFF file.
-2017.07.11
- Add 'out' parameter to CziFile.asarray.
- Remove memmap option from CziFile.asarray (backwards incompatible).
- Change spline interpolation order to 0 (backwards incompatible).
- Make axes return a string.
- Require tifffile 2017.07.11.
-2015.08.17
- Require tifffile 2015.08.17.
-2014.10.10
- Read data into a memory mapped array (optional).
-2013.12.04
- Decode JpegXrFile and JpgFile via _czifle extension module.
- Attempt to reconstruct tiled mosaic images.
-2013.11.20
- Initial release.
-
-Notes
------
-The API is not stable yet and might change between revisions.
-
-The file format design specification [1] is confidential and the licence
-agreement does not permit to write data into CZI files.
-
-Only a subset of the 2012 specification is implemented in the initial release.
-Specifically, multifile images are not yet supported.
-
-Tested on Windows with a few example files only.
-
-References
-----------
-(1) ZISRAW (CZI) File Format Design specification Release Version 1.2.2.
- CZI 07-2016/CZI-DOC ZEN 2.3/DS_ZISRAW-FileFormat.pdf (confidential).
- Documentation can be requested at
-
-(2) CZI The File Format for the Microscope | ZEISS International
-
-
-Examples
---------
->>> with CziFile('test.czi') as czi:
-... image = czi.asarray()
->>> image.shape
-(3, 3, 3, 250, 200, 3)
->>> image[0, 0, 0, 0, 0]
-array([10, 10, 10], dtype=uint8)
-
-"""
-
-from __future__ import division, print_function
-
-import os
-import sys
-import re
-import uuid
-import time
-import struct
-import warnings
-import multiprocessing
-
-from concurrent.futures import ThreadPoolExecutor
-
-try:
- from lxml import etree
-except ImportError:
- from xml.etree import cElementTree as etree
-
-import numpy
-from scipy.ndimage.interpolation import zoom
-
-from tifffile import (FileHandle, memmap, decode_lzw, lazyattr, repeat_nd,
- product, stripnull, format_size, squeeze_axes,
- create_output)
-
-try:
- if __package__:
- from aicsimageio import _czifile
- else:
- import _czifile
-except ImportError:
- warnings.warn(
- "ImportError: No module named '_czifile'. "
- "Decoding of JXR and JPEG encoded images will not be available. "
- "Czifile.pyx can be obtained at http://www.lfd.uci.edu/~gohlke/")
- _czifile = None
-
-__version__ = '2017.09.12'
-__docformat__ = 'restructuredtext en'
-__all__ = 'imread', 'CziFile'
-
-
-def imread(filename, *args, **kwargs):
- """Return image data from CZI file as numpy array.
-
- 'args' and 'kwargs' are arguments to the CziFile.asarray function.
-
- Examples
- --------
- >>> image = imread('test.czi')
- >>> image.shape
- (3, 3, 3, 250, 200, 3)
- >>> image.dtype
- dtype('uint8')
-
- """
- with CziFile(filename) as czi:
- result = czi.asarray(*args, **kwargs)
- return result
-
-
-class CziFile(object):
- """Carl Zeiss Image (CZI) file.
-
- Attributes
- ----------
- header : FileHeaderSegment
- Global file metadata such as file version and GUID.
- metadata : etree.ElementTree.Element
- Global image metadata in UTF-8 encoded XML format.
-
- All attributes are read-only.
-
- """
- def __init__(self, arg, multifile=True, filesize=None, detectmosaic=True):
- """Open CZI file and read header.
-
- Raise ValueError if file is not a ZISRAW file.
-
- Parameters
- ----------
- multifile : bool
- If True (default), the master file of a multifile CZI file
- will be opened if applicable.
- filesize : int
- Size of file if arg is a file handle pointing to an
- embedded CZI file.
- detectmosaic : bool
- If True (default), mosaic images will be reconstructed from
- SubBlocks with a tile index.
-
- Notes
- -----
- CziFile instances created from file name must be closed using the
- 'close' method, which is automatically called when using the
- 'with' statement.
-
- """
- self._fh = FileHandle(arg, size=filesize)
- try:
- if self._fh.read(10) != b'ZISRAWFILE':
- raise ValueError("not a CZI file")
- self.header = Segment(self._fh, 0).data()
- except Exception:
- self._fh.close()
- raise
-
- if multifile and self.header.file_part and isinstance(arg, basestring):
- # open master file instead
- self._fh.close()
- name, _ = match_filename(arg)
- self._fh = FileHandle(name)
- self.header = Segment(self._fh, 0).data()
- assert(self.header.primary_file_guid == self.header.file_guid)
- assert(self.header.file_part == 0)
-
- if self.header.update_pending:
- warnings.warn("file is pending update")
- self._filter_mosaic = detectmosaic
-
- def segments(self, kind=None):
- """Return iterator over Segment data of specified kind.
-
- Parameters
- ----------
- kind : bytestring or sequence thereof
- Segment id(s) as listed in SEGMENT_ID.
- If None (default), all segments are returned.
-
- """
- fpos = 0
- while True:
- self._fh.seek(fpos)
- try:
- segment = Segment(self._fh)
- except SegmentNotFoundError:
- break
- if (kind is None) or (segment.sid in kind):
- yield segment.data()
- fpos = segment.data_offset + segment.allocated_size
-
- @lazyattr
- def metadata(self):
- """Return data from MetadataSegment as xml.ElementTree root Element.
-
- Return None if no Metadata segment is found.
-
- """
- if self.header.metadata_position:
- segment = Segment(self._fh, self.header.metadata_position)
- if segment.sid == MetadataSegment.SID:
- data = segment.data().data()
- return etree.fromstring(data.encode('utf-8'))
- warnings.warn("Metadata segment not found")
- try:
- metadata = next(self.segments(MetadataSegment.SID))
- return etree.fromstring(metadata.data().encode('utf-8'))
- except StopIteration:
- pass
-
- @lazyattr
- def subblock_directory(self):
- """Return list of all DirectoryEntryDV in file.
-
- Use SubBlockDirectorySegment if exists, else find SubBlockSegments.
-
- """
- if self.header.directory_position:
- segment = Segment(self._fh, self.header.directory_position)
- if segment.sid == SubBlockDirectorySegment.SID:
- return segment.data().entries
- warnings.warn("SubBlockDirectory segment not found")
- return list(segment.directory_entry for segment in
- self.segments(SubBlockSegment.SID))
-
- @lazyattr
- def attachment_directory(self):
- """Return list of all AttachmentEntryA1 in file.
-
- Use AttachmentDirectorySegment if exists, else find AttachmentSegments.
-
- """
- if self.header.attachment_directory_position:
- segment = Segment(self._fh,
- self.header.attachment_directory_position)
- if segment.sid == AttachmentDirectorySegment.SID:
- return segment.data().entries
- warnings.warn("AttachmentDirectory segment not found")
- return list(segment.attachment_entry for segment in
- self.segments(AttachmentSegment.SID))
-
- def subblocks(self):
- """Return iterator over all SubBlock segments in file."""
- for entry in self.subblock_directory:
- yield entry.data_segment()
-
- def attachments(self):
- """Return iterator over all Attachment segments in file."""
- for entry in self.attachment_directory:
- yield entry.data_segment()
-
- def save_attachments(self, directory=None):
- """Save all attachments to files."""
- if directory is None:
- directory = self._fh.path + '.attachments'
- if not os.path.exists(directory):
- os.makedirs(directory)
- for attachment in self.attachments():
- attachment.save(directory=directory)
-
- @lazyattr
- def filtered_subblock_directory(self):
- """Return sorted list of DirectoryEntryDV if mosaic, else all."""
- if not self._filter_mosaic:
- return self.subblock_directory
- filtered = [directory_entry
- for directory_entry in self.subblock_directory
- if directory_entry.mosaic_index is not None]
- if not filtered:
- return self.subblock_directory
- return list(sorted(filtered, key=lambda x: x.mosaic_index))
-
- @lazyattr
- def shape(self):
- """Return shape of image data in file."""
- shape = [[dim.start + dim.size
- for dim in directory_entry.dimension_entries
- if dim.dimension != b'M']
- for directory_entry in self.filtered_subblock_directory]
- shape = numpy.max(shape, axis=0)
- shape = tuple(int(i-j) for i, j in zip(shape, self.start[:-1]))
- dtype = self.filtered_subblock_directory[0].dtype
- sampleshape = numpy.dtype(dtype).shape
- shape = shape + (sampleshape if sampleshape else (1,))
- return shape
-
- @lazyattr
- def start(self):
- """Return minimum start indices per dimension of sub images in file."""
- start = [[dim.start
- for dim in directory_entry.dimension_entries
- if dim.dimension != b'M']
- for directory_entry in self.filtered_subblock_directory]
- start = tuple(numpy.min(start, axis=0)) + (0,)
- return start
-
- @lazyattr
- def axes(self):
- """Return axes of image data in file."""
- return self.filtered_subblock_directory[0].axes
-
- @lazyattr
- def dtype(self):
- """Return numpy dtype of image data in file."""
- # subblock data can be of different pixel type
- dtype = numpy.dtype(self.filtered_subblock_directory[0].dtype[-2:])
- for directory_entry in self.filtered_subblock_directory:
- dtype = numpy.promote_types(dtype, directory_entry.dtype[-2:])
- return dtype
-
- def asarray(self, resize=True, order=0, out=None, max_workers=None):
- """Return image data from file(s) as numpy array.
-
- Parameters
- ----------
- resize : bool
- If True (default), resize sub/supersampled subblock data.
- order : int
- The order of spline interpolation used to resize sub/supersampled
- subblock data. Default is 0 (nearest neighbor).
- out : numpy.ndarray, str, or file-like object; optional
- Buffer where image data will be saved.
- If numpy.ndarray, a writable array of compatible dtype and shape.
- If str or open file, the file name or file object used to
- create a memory-map to an array stored in a binary file on disk.
- max_workers : int
- Maximum number of threads to read and decode subblock data.
- By default up to half the CPU cores are used.
-
- """
- out = create_output(out, self.shape, self.dtype)
-
- if max_workers is None:
- max_workers = multiprocessing.cpu_count() // 2
-
- def func(directory_entry, resize=resize, order=order,
- start=self.start, out=out):
- """Read, decode, and copy subblock data."""
- subblock = directory_entry.data_segment()
- tile = subblock.data(resize=resize, order=order)
- index = [slice(i-j, i-j+k) for i, j, k in
- zip(directory_entry.start, start, tile.shape)]
- try:
- out[index] = tile
- except ValueError as e:
- warnings.warn(str(e))
-
- if max_workers > 1:
- self._fh.lock = True
- with ThreadPoolExecutor(max_workers) as executor:
- executor.map(func, self.filtered_subblock_directory)
- self._fh.lock = None
- else:
- list(map(func, self.filtered_subblock_directory))
-
- if hasattr(out, 'flush'):
- out.flush()
- return out
-
- def close(self):
- self._fh.close()
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- self.close()
-
- def __str__(self):
- return '\n '.join((
- self._fh.name.capitalize(),
- "(Carl Zeiss Image File)",
- str(self.header),
- "MetadataSegment",
- str(self.axes),
- str(self.shape),
- str(self.dtype),
- str(etree.tostring(self.metadata))))
-
-
-class Segment(object):
- """ZISRAW Segment."""
-
- __slots__ = 'sid', 'allocated_size', 'used_size', 'data_offset', '_fh'
-
- def __init__(self, fh, fpos=None):
- """Read segment header from file."""
- if fpos is not None:
- fh.seek(fpos)
- try:
- (self.sid,
- self.allocated_size,
- self.used_size
- ) = struct.unpack('<16sqq', fh.read(32))
- except struct.error:
- raise SegmentNotFoundError("can not read ZISRAW segment")
- self.sid = stripnull(self.sid)
- if self.sid not in SEGMENT_ID:
- if not self.sid.startswith(b'ZISRAW'):
- raise SegmentNotFoundError("not a ZISRAW segment")
- warnings.warn("unknown segment type %s" % self.sid)
- self.data_offset = fh.tell()
- self._fh = fh
-
- def data(self):
- """Read segment data from file and return as *Segment instance."""
- self._fh.seek(self.data_offset)
- return SEGMENT_ID.get(self.sid, UnknownSegment)(self._fh)
-
- def __str__(self):
- return "Segment %s %i of %i" % (
- self.sid, self.used_size, self.allocated_size)
-
-
-class SegmentNotFoundError(Exception):
- """Exception to indicate that file position does not contain Segment."""
- pass
-
-
-class FileHeaderSegment(object):
- """ZISRAWFILE file header segment data.
-
- Contains global file metadata such as file version and GUID.
-
- """
- __slots__ = ('version', 'primary_file_guid', 'file_guid',
- 'file_part', 'directory_position', 'metadata_position',
- 'update_pending', 'attachment_directory_position')
-
- SID = b'ZISRAWFILE'
-
- def __init__(self, fh):
- (major,
- minor,
- reserved1,
- reserved2,
- primary_file_guid,
- file_guid,
- self.file_part,
- self.directory_position,
- self.metadata_position,
- self.update_pending,
- self.attachment_directory_position,
- ) = struct.unpack(' 1 else 0
- return name, part
-
-
-def decode_jxr(data):
- """Decode JXR data stream into ndarray."""
- return _czifile.decode_jxr(data)
-
-
-def decode_jpeg(data):
- """Decode JPEG data stream into ndarray."""
- return _czifile.decode_jpeg(data)
-
-
-# map Segment.sid to data reader
-SEGMENT_ID = {
- FileHeaderSegment.SID: FileHeaderSegment,
- SubBlockDirectorySegment.SID: SubBlockDirectorySegment,
- SubBlockSegment.SID: SubBlockSegment,
- MetadataSegment.SID: MetadataSegment,
- AttachmentSegment.SID: AttachmentSegment,
- AttachmentDirectorySegment.SID: AttachmentDirectorySegment,
- DeletedSegment.SID: DeletedSegment,
-}
-
-# map AttachmentEntryA1.content_file_type to attachment reader.
-CONTENT_FILE_TYPE = {
- b'CZI': CziFile,
- b'ZISRAW': CziFile,
- b'CZTIMS': TimeStamps,
- b'CZEVL': EventList,
- b'CZLUT': LookupTables,
- b'CZFOC': FocusPositions,
- b'CZEXP': xml_reader, # Experiment
- b'CZHWS': xml_reader, # HardwareSetting
- b'CZMVM': xml_reader, # MultiviewMicroscopy
- # b'CZPML': PalMoleculeList, # undocumented
- # b'ZIP'
- # b'JPG'
-}
-
-# map DirectoryEntryDV.pixeltype to numpy dtypes
-PIXEL_TYPE = {
- 0: '`_
-
-:Organization:
- Laboratory for Fluorescence Dynamics, University of California, Irvine
-
-:Version: 2017.07.20
-
-Requirements
-------------
-* `CPython 3.6 `_
-* `Numpy 1.13 `_
-* `Cython 0.25 `_ (build)
-* `jpeglib v9 `_ (build)
-* `jxrlib 0.2.0 `_ (build)
-* `jxrlib patch `_ (build)
-* A Python distutils compatible C compiler (build)
-
-Install
--------
-Use this Cython distutils setup script to build the extension module::
-
- # setup.py
- # Usage: ``python setup.py build_ext --inplace``
- import sys
- import os
- from distutils.core import setup, Extension
- from Cython.Distutils import build_ext
- jxrlib_dir = 'jxrlib' # directory where jxrlib was built
- jpeg_dir = 'jpeg-9'
- win32 = sys.platform == 'win32'
- include_dirs = [jpeg_dir]
- include_dirs += [os.path.join(jxrlib_dir, *d.split('/'))
- for d in ('jxrgluelib', 'common/include', 'image/sys')]
- define_macros = [('WIN32', None)] if win32 else [('INITGUID', None)]
- ext = Extension('_czifile', sources=['_czifile.pyx'],
- include_dirs=include_dirs, define_macros=define_macros,
- library_dirs=[jxrlib_dir, jpeg_dir],
- libraries=['jxrlib' if win32 else 'libjpegxr',
- 'jpeg'],)
- setup(name='_czifile', cmdclass={'build_ext': build_ext}, ext_modules=[ext])
-
-"""
-
-__version__ = "2017.07.20"
-
-cimport cython
-from cython.operator cimport dereference as deref
-from libc.setjmp cimport setjmp, longjmp, jmp_buf
-cimport numpy
-import numpy
-
-# jxrlib
-
-cdef extern from "windowsmediaphoto.h":
- int WMP_errSuccess
- int WMP_errFail
- int WMP_errNotYetImplemented
- int WMP_errAbstractMethod
- int WMP_errOutOfMemory
- int WMP_errFileIO
- int WMP_errBufferOverflow
- int WMP_errInvalidParameter
- int WMP_errInvalidArgument
- int WMP_errUnsupportedFormat
- int WMP_errIncorrectCodecVersion
- int WMP_errIndexNotFound
- int WMP_errOutOfSequence
- int WMP_errNotInitialized
- int WMP_errMustBeMultipleOf16LinesUntilLastCall
- int WMP_errPlanarAlphaBandedEncRequiresTempFile
- int WMP_errAlphaModeCannotBeTranscoded
- int WMP_errIncorrectCodecSubVersion
-
- ctypedef long ERR
- ctypedef int I32
- ctypedef int PixelI
- ctypedef unsigned char U8
- ctypedef unsigned int U32
-
-
-cdef extern from "guiddef.h":
- ctypedef struct GUID:
- pass
-
- int IsEqualGUID(GUID*, GUID*)
-
-
-cdef extern from "JXRGlue.h":
- ctypedef GUID PKPixelFormatGUID
-
- ctypedef struct PKFactory:
- pass
- ctypedef struct PKCodecFactory:
- pass
- ctypedef struct PKImageDecode:
- pass
- ctypedef struct PKImageEncode:
- pass
- ctypedef struct PKFormatConverter:
- pass
- ctypedef struct PKRect:
- I32 X, Y, Width, Height
-
- cdef ERR PKCreateCodecFactory(PKCodecFactory**, U32) nogil
- cdef ERR PKCreateCodecFactory_Release(PKCodecFactory**) nogil
- cdef ERR PKCodecFactory_CreateFormatConverter(PKFormatConverter**) nogil
- cdef ERR PKCodecFactory_CreateDecoderFromBytes(void*, size_t,
- PKImageDecode**) nogil
- cdef ERR PKImageDecode_GetSize(PKImageDecode*, I32*, I32*) nogil
- cdef ERR PKImageDecode_Release(PKImageDecode**) nogil
- cdef ERR PKImageDecode_GetPixelFormat(PKImageDecode*,
- PKPixelFormatGUID*) nogil
- cdef ERR PKFormatConverter_Release(PKFormatConverter**) nogil
- cdef ERR PKFormatConverter_Initialize(PKFormatConverter*, PKImageDecode*,
- char*, PKPixelFormatGUID) nogil
- cdef ERR PKFormatConverter_Copy(PKFormatConverter*, const PKRect*,
- U8*, U32) nogil
- cdef ERR PKFormatConverter_Convert(PKFormatConverter*, const PKRect*,
- U8*, U32) nogil
-
- GUID GUID_PKPixelFormat8bppGray
- GUID GUID_PKPixelFormat16bppGray
- GUID GUID_PKPixelFormat32bppGrayFloat
- GUID GUID_PKPixelFormat24bppBGR
- GUID GUID_PKPixelFormat24bppRGB
- GUID GUID_PKPixelFormat48bppRGB
- GUID GUID_PKPixelFormat128bppRGBFloat
- GUID GUID_PKPixelFormat32bppRGBA
- GUID GUID_PKPixelFormat32bppBGRA
- GUID GUID_PKPixelFormat64bppRGBA
- GUID GUID_PKPixelFormat128bppRGBAFloat
-
-
-WMP_ERR = {
- WMP_errFail: "Fail",
- WMP_errNotYetImplemented: "NotYetImplemented",
- WMP_errAbstractMethod: "AbstractMethod",
- WMP_errOutOfMemory: "OutOfMemory",
- WMP_errFileIO: "FileIO",
- WMP_errBufferOverflow: "BufferOverflow",
- WMP_errInvalidParameter: "InvalidParameter",
- WMP_errInvalidArgument: "InvalidArgument",
- WMP_errUnsupportedFormat: "UnsupportedFormat",
- WMP_errIncorrectCodecVersion: "IncorrectCodecVersion",
- WMP_errIndexNotFound: "IndexNotFound",
- WMP_errOutOfSequence: "OutOfSequence",
- WMP_errNotInitialized: "NotInitialized",
- WMP_errAlphaModeCannotBeTranscoded: "AlphaModeCannotBeTranscoded",
- WMP_errIncorrectCodecSubVersion: "IncorrectCodecSubVersion",
- WMP_errMustBeMultipleOf16LinesUntilLastCall:
- "MustBeMultipleOf16LinesUntilLastCall",
- WMP_errPlanarAlphaBandedEncRequiresTempFile:
- "PlanarAlphaBandedEncRequiresTempFile",
- }
-
-
-class WmpError(Exception):
- def __init__(self, msg, err):
- msg = "%s failed with %s error" % (msg, WMP_ERR.get(err, "Unknown"))
- Exception.__init__(self, msg)
-
-
-def decode_jxr(data):
- """Return image data from JXR bytes as numpy array."""
- cdef unsigned char* cdata = data
- cdef numpy.ndarray out
- cdef PKImageDecode* decoder = NULL
- cdef PKFormatConverter* converter = NULL
- cdef PKPixelFormatGUID pixel_format
- cdef PKRect rect
- cdef I32 width
- cdef I32 height
- cdef U32 stride
- cdef ERR err
-
- try:
- err = PKCodecFactory_CreateDecoderFromBytes(cdata, len(data), &decoder)
- if err:
- raise WmpError("PKCodecFactory_CreateDecoderFromBytes", err)
-
- err = PKImageDecode_GetSize(decoder, &width, &height)
- if err:
- raise WmpError("PKImageDecode_GetSize", err)
-
- err = PKImageDecode_GetPixelFormat(decoder, &pixel_format)
- if err:
- raise WmpError("PKImageDecode_GetPixelFormat", err)
-
- if IsEqualGUID(&pixel_format, &GUID_PKPixelFormat8bppGray):
- dtype = numpy.uint8
- samples = 1
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat16bppGray):
- dtype = numpy.uint16
- samples = 1
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat32bppGrayFloat):
- dtype = numpy.float32
- samples = 1
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat24bppBGR):
- dtype = numpy.uint8
- samples = 3
- pixel_format = GUID_PKPixelFormat24bppRGB
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat24bppRGB):
- dtype = numpy.uint8
- samples = 3
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat48bppRGB):
- dtype = numpy.uint16
- samples = 3
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat128bppRGBFloat):
- dtype = numpy.float32
- samples = 3
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat32bppBGRA):
- dtype = numpy.uint8
- samples = 4
- pixel_format = GUID_PKPixelFormat32bppRGBA
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat32bppRGBA):
- dtype = numpy.uint8
- samples = 4
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat64bppRGBA):
- dtype = numpy.uint16
- samples = 4
- elif IsEqualGUID(&pixel_format, &GUID_PKPixelFormat128bppRGBAFloat):
- dtype = numpy.float32
- samples = 4
- else:
- raise ValueError("unknown pixel format")
-
- err = PKCodecFactory_CreateFormatConverter(&converter)
- if err:
- raise WmpError("PKCodecFactory_CreateFormatConverter", err)
-
- err = PKFormatConverter_Initialize(converter, decoder, NULL,
- pixel_format)
- if err:
- raise WmpError("PKFormatConverter_Initialize", err)
-
- shape = height, width
- if samples > 1:
- shape += samples,
- out = numpy.empty(shape, dtype)
-
- stride = out.strides[0]
- rect.X = 0
- rect.Y = 0
- rect.Width = out.shape[1]
- rect.Height = out.shape[0]
-
- # TODO: check alignment issues
- with nogil:
- err = PKFormatConverter_Copy(converter, &rect, out.data,
- stride)
- if err:
- raise WmpError("PKFormatConverter_Copy", err)
-
- return out
-
- finally:
- if converter != NULL:
- PKFormatConverter_Release(&converter)
- if decoder != NULL:
- PKImageDecode_Release(&decoder)
-
-# jpeglib
-
-cdef extern from "jpeglib.h":
- ctypedef void noreturn_t
- ctypedef int boolean
- ctypedef unsigned int JDIMENSION
- ctypedef unsigned char JSAMPLE
- ctypedef JSAMPLE* JSAMPROW
- ctypedef JSAMPROW* JSAMPARRAY
-
- cdef enum J_COLOR_SPACE:
- JCS_UNKNOWN,
- JCS_GRAYSCALE,
- JCS_RGB,
- JCS_YCbCr,
- JCS_CMYK,
- JCS_YCCK
-
- cdef enum J_DITHER_MODE:
- JDITHER_NONE,
- JDITHER_ORDERED,
- JDITHER_FS
-
- cdef enum J_DCT_METHOD:
- JDCT_ISLOW,
- JDCT_IFAST,
- JDCT_FLOAT
-
- cdef struct jpeg_source_mgr:
- pass
-
- cdef struct jpeg_common_struct:
- jpeg_error_mgr* err
-
- cdef struct jpeg_error_mgr:
- int msg_code
- char** jpeg_message_table
- noreturn_t error_exit(jpeg_common_struct*)
- void output_message(jpeg_common_struct*)
-
- cdef struct jpeg_decompress_struct:
- jpeg_error_mgr* err
- void* client_data
- jpeg_source_mgr* src
- JDIMENSION image_width
- JDIMENSION image_height
- JDIMENSION output_width
- JDIMENSION output_height
- JDIMENSION output_scanline
- J_COLOR_SPACE jpeg_color_space
- J_COLOR_SPACE out_color_space
- J_DCT_METHOD dct_method
- J_DITHER_MODE dither_mode
- boolean buffered_image
- boolean raw_data_out
- boolean do_fancy_upsampling
- boolean do_block_smoothing
- boolean quantize_colors
- boolean two_pass_quantize
- unsigned int scale_num
- unsigned int scale_denom
- int num_components
- int out_color_components
- int output_components
- int rec_outbuf_height
- int desired_number_of_colors
- int actual_number_of_colors
- int data_precision
- double output_gamma
-
- cdef jpeg_error_mgr* jpeg_std_error(jpeg_error_mgr*) nogil
- cdef void jpeg_create_decompress(jpeg_decompress_struct*) nogil
- cdef void jpeg_destroy_decompress(jpeg_decompress_struct*) nogil
- cdef void jpeg_mem_src(jpeg_decompress_struct*,
- unsigned char*, unsigned long) nogil
- cdef int jpeg_read_header(jpeg_decompress_struct*, boolean) nogil
- cdef boolean jpeg_start_decompress(jpeg_decompress_struct*) nogil
- cdef boolean jpeg_finish_decompress (jpeg_decompress_struct*) nogil
- cdef JDIMENSION jpeg_read_scanlines(jpeg_decompress_struct*,
- JSAMPARRAY, JDIMENSION) nogil
-
-
-ctypedef struct my_error_mgr:
- jpeg_error_mgr pub
- jmp_buf setjmp_buffer
-
-
-cdef void my_error_exit(jpeg_common_struct* cinfo):
- cdef my_error_mgr* error = deref(cinfo).err
- longjmp(deref(error).setjmp_buffer, 1)
-
-
-cdef void my_output_message(jpeg_common_struct* cinfo):
- pass
-
-
-class JpgError(Exception):
- pass
-
-
-def decode_jpeg(data, tables=b''):
- """Return image data from in memory JPG file as numpy array."""
- cdef numpy.ndarray out
- cdef int width
- cdef int height
- cdef int numsamples
- cdef my_error_mgr err
- cdef jpeg_decompress_struct cinfo
- cdef JSAMPROW samples
- cdef unsigned char* cdata = data
- cdef unsigned char* ctables = tables
-
- cinfo.err = jpeg_std_error(&err.pub)
- err.pub.error_exit = my_error_exit
- err.pub.output_message = my_output_message
- if setjmp(err.setjmp_buffer):
- jpeg_destroy_decompress(&cinfo)
- raise JpgError(err.pub.jpeg_message_table[err.pub.msg_code])
-
- jpeg_create_decompress(&cinfo)
- cinfo.do_fancy_upsampling = True
-
- if len(tables) > 0:
- jpeg_mem_src(&cinfo, ctables, len(tables))
- jpeg_read_header(&cinfo, 0)
-
- jpeg_mem_src(&cinfo, cdata, len(data))
- jpeg_read_header(&cinfo, 1)
-
- shape = cinfo.image_height, cinfo.image_width
- if cinfo.num_components > 1:
- shape += cinfo.num_components,
- out = numpy.empty(shape, numpy.uint8)
-
- with nogil:
- jpeg_start_decompress(&cinfo)
-
- samples = out.data
- while cinfo.output_scanline < cinfo.output_height:
- numsamples = jpeg_read_scanlines(&cinfo, &samples, 1)
- samples += numsamples * cinfo.image_width * cinfo.num_components
-
- jpeg_finish_decompress(&cinfo)
- jpeg_destroy_decompress(&cinfo)
-
- return out
diff --git a/aicsimageio/writers/__init__.py b/aicsimageio/writers/__init__.py
index 082cb9ebf..b2e939563 100644
--- a/aicsimageio/writers/__init__.py
+++ b/aicsimageio/writers/__init__.py
@@ -1,2 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
from .ome_tiff_writer import OmeTiffWriter # noqa: F401
from .png_writer import PngWriter # noqa: F401
diff --git a/examples/local_cluster.ipynb b/examples/local_cluster.ipynb
new file mode 100644
index 000000000..07ac6aa8c
--- /dev/null
+++ b/examples/local_cluster.ipynb
@@ -0,0 +1,518 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "