From a24f0a3e260c6a9d40f14b6a1a2c4240f3562470 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Sun, 17 Oct 2021 20:23:47 -0500 Subject: [PATCH 01/14] Add logic to tile dask images from bioformats reader --- aicsimageio/readers/bioformats_reader.py | 71 ++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index 1a2bce3e1..e4b671198 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -6,7 +6,7 @@ from functools import lru_cache from pathlib import Path from threading import Lock -from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union, List import dask.array as da import numpy as np @@ -101,6 +101,8 @@ def __init__( original_meta: bool = False, memoize: Union[int, bool] = 0, options: Dict[str, bool] = {}, + dask_tiles: bool = False, + tile_size: Optional[Tuple[int, int]] = None, ): self._fs, self._path = io_utils.pathlike_to_fs(image, enforce_exists=True) # Catch non-local file system @@ -114,6 +116,8 @@ def __init__( "options": options, "original_meta": original_meta, "memoize": memoize, + "dask_tiles": dask_tiles, + "tile_size": tile_size, } try: with BioFile(self._path, **self._bf_kwargs) as rdr: # type: ignore @@ -258,6 +262,8 @@ def __init__( original_meta: bool = False, memoize: Union[int, bool] = 0, options: Dict[str, bool] = {}, + dask_tiles: bool = False, + tile_size: int = None, ): try: loci = get_loci() @@ -299,6 +305,16 @@ def __init__( self._lock = Lock() self.set_series(series) + self.dask_tiles = dask_tiles + if self.dask_tiles: + if tile_size is None: + self.tile_size = ( + self._r.getOptimalTileHeight(), + self._r.getOptimalTileWidth(), + ) + else: + self.tile_size = tile_size + def set_series(self, series: int = 0) -> None: self._r.setSeries(series) self._core_meta = CoreMeta( @@ -367,7 +383,12 @@ def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: self._r.setSeries(series) nt, nc, nz, ny, nx, nrgb = self.core_meta.shape - chunks = ((1,) * nt, (1,) * nc, (1,) * nz, ny, nx) + + if self.dask_tiles: + chunks = _get_dask_tile_chunks(nt, nc, nz, ny, nx, self.tile_size) + else: + chunks = ((1,) * nt, (1,) * nc, (1,) * nz, ny, nx) + if nrgb > 1: chunks = chunks + (nrgb,) # type: ignore arr = da.map_blocks( @@ -481,8 +502,16 @@ def _dask_chunk(self, block_id: Tuple[int, ...]) -> np.ndarray: """ # Our convention is that the final dask array is in the order TCZYX, so # block_id will be coming in as (T, C, Z, Y, X). - t, c, z, *_ = block_id - im = self._get_plane(t, c, z) + t, c, z, y, x, *_ = block_id + + if self.dask_tiles: + *_, ny, nx, _ = self.core_meta.shape + y_slice = _axis_id_to_slice(y, self.tile_size[0], ny) + x_slice = _axis_id_to_slice(x, self.tile_size[1], nx) + im = self._get_plane(t, c, z, y_slice, x_slice) + else: + im = self._get_plane(t, c, z) + return im[np.newaxis, np.newaxis, np.newaxis] _service: Any = None @@ -517,6 +546,40 @@ def _pixtype2dtype(pixeltype: int, little_endian: bool) -> np.dtype: return np.dtype(("<" if little_endian else ">") + fmt2type[pixeltype]) +def _get_dask_tile_chunks( + nt: int, nc: int, nz: int, ny: int, nx: int, tile_size: Tuple[int, int] +) -> Tuple[Tuple, Tuple, Tuple, Tuple, Tuple]: + """Returns chunking tuples (length of each chunk in each axis) after tiling. + I.e., if nx == 2048 and tile_size == 1024, chunks for x axis will be (1024,1024)""" + + y_tile_size, x_tile_size = tile_size + + def _chunk_by_tile_size(n_px, tile_length): + n_splits = n_px / tile_length + n_full_tiles = np.floor(n_splits) + + if n_splits.is_integer(): + tile_chunks = (int(tile_length),) * int(n_full_tiles) + else: + edge_tile = n_px - (n_full_tiles * tile_length) + tile_chunks = (int(tile_length),) * int(n_full_tiles) + (int(edge_tile),) + return tile_chunks + + y_tiling_chunks = _chunk_by_tile_size(ny, y_tile_size) + x_tiling_chunks = _chunk_by_tile_size(nx, x_tile_size) + + return ((1,) * nt, (1,) * nc, (1,) * nz, y_tiling_chunks, x_tiling_chunks) + + +def _axis_id_to_slice(axis_id: int, tile_length: int, n_px: int) -> slice: + """Take the axis_id from a dask block_id and create the corresponding + tile slice, taking into account edge tiles.""" + if (axis_id * tile_length) + tile_length <= n_px: + return slice(axis_id * tile_length, (axis_id * tile_length) + tile_length) + else: + return slice(axis_id * tile_length, n_px) + + def _slice2width(slc: slice, length: int) -> Tuple[int, int]: """Convert `slice` object into (start, width)""" if slc.stop is not None or slc.start is not None: From 6b752308e8e95ce0f193ccc5674c66934cf8f0aa Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Sun, 17 Oct 2021 20:25:21 -0500 Subject: [PATCH 02/14] remove unused type check type --- aicsimageio/readers/bioformats_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index e4b671198..fe5c428cf 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -6,7 +6,7 @@ from functools import lru_cache from pathlib import Path from threading import Lock -from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union, List +from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union import dask.array as da import numpy as np From e99c4b4384dfd1ae2d7c946f0d2f281f0d240134 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Sun, 17 Oct 2021 20:46:19 -0500 Subject: [PATCH 03/14] doc strings --- aicsimageio/readers/bioformats_reader.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index fe5c428cf..1a7aedd09 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -75,7 +75,13 @@ class BioformatsReader(Reader): see: https://docs.openmicroscopy.org/bio-formats/latest/formats/options.html For example: to turn off chunkmap table reading for ND2 files, use `options={"nativend2.chunkmap": False}` - + dask_tiles: bool, optional + Whether to chunk the bioformats dask array by tiles to easily read sub-regions + with numpy-like array indexing + Defaults to false and iamges are read by entire planes + tile_size: Optional[Tuple[int, int]] + Tuple that sets the tile size of y and x axis, respectively + By default, it will use optimal values computed by bioformats itself Raises ------ exceptions.UnsupportedFileFormatError @@ -251,6 +257,13 @@ class BioFile: see: https://docs.openmicroscopy.org/bio-formats/latest/formats/options.html For example: to turn off chunkmap table reading for ND2 files, use `options={"nativend2.chunkmap": False}` + dask_tiles: bool, optional + Whether to chunk the bioformats dask array by tiles to easily read sub-regions + with numpy-like array indexing + Defaults to false and iamges are read by entire planes + tile_size: Optional[Tuple[int, int]] + Tuple that sets the tile size of y and x axis, respectively + By default, it will use optimal values computed by bioformats itself """ def __init__( From acf6a97f1aef589b23412551ce6358dae36f1269 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Mon, 18 Oct 2021 08:41:12 -0500 Subject: [PATCH 04/14] switch location of args --- aicsimageio/readers/bioformats_reader.py | 36 ++++++++++++++++-------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index 1a7aedd09..0b32b9152 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -257,13 +257,6 @@ class BioFile: see: https://docs.openmicroscopy.org/bio-formats/latest/formats/options.html For example: to turn off chunkmap table reading for ND2 files, use `options={"nativend2.chunkmap": False}` - dask_tiles: bool, optional - Whether to chunk the bioformats dask array by tiles to easily read sub-regions - with numpy-like array indexing - Defaults to false and iamges are read by entire planes - tile_size: Optional[Tuple[int, int]] - Tuple that sets the tile size of y and x axis, respectively - By default, it will use optimal values computed by bioformats itself """ def __init__( @@ -275,8 +268,6 @@ def __init__( original_meta: bool = False, memoize: Union[int, bool] = 0, options: Dict[str, bool] = {}, - dask_tiles: bool = False, - tile_size: int = None, ): try: loci = get_loci() @@ -376,7 +367,9 @@ def to_numpy(self, series: Optional[int] = None) -> np.ndarray: """ return np.asarray(self.to_dask(series)) - def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: + def to_dask(self, series: Optional[int] = None, + dask_tiles: bool = False, + tile_size: Optional[Tuple[int,int]] = None,) -> DaskArrayProxy: """Create dask array for the specified or current series. Note: the order of the returned array will *always* be `TCZYX[r]`, @@ -388,6 +381,18 @@ def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: has all the methods and behavior of a dask array. see :class:`aicsimageio.utils.dask_proxy.DaskArrayProxy`. + Parameters + ---------- + series : int, optional + The series index to retrieve, by default None + dask_tiles: bool, optional + Whether to chunk the bioformats dask array by tiles to easily read sub-regions + with numpy-like array indexing + Defaults to false and iamges are read by entire planes + tile_size: Optional[Tuple[int, int]] + Tuple that sets the tile size of y and x axis, respectively + By default, it will use optimal values computed by bioformats itself + Returns ------- DaskArrayProxy @@ -397,8 +402,15 @@ def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: nt, nc, nz, ny, nx, nrgb = self.core_meta.shape - if self.dask_tiles: - chunks = _get_dask_tile_chunks(nt, nc, nz, ny, nx, self.tile_size) + if dask_tiles: + if tile_size is None: + yx_tile_size = ( + self._r.getOptimalTileHeight(), + self._r.getOptimalTileWidth(), + ) + else: + yx_tile_size = tile_size + chunks = _get_dask_tile_chunks(nt, nc, nz, ny, nx, yx_tile_size) else: chunks = ((1,) * nt, (1,) * nc, (1,) * nz, ny, nx) From 1b14b5a6e6cb951ac1325743f717d566961a8d3c Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Mon, 18 Oct 2021 13:51:54 -0500 Subject: [PATCH 05/14] Add an M dimension so `reconstruct_mosaic=True` returns the tile-chunked version of the dask image --- aicsimageio/readers/bioformats_reader.py | 113 ++++++++++++++++------- 1 file changed, 80 insertions(+), 33 deletions(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index 0b32b9152..5e8e1f501 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -75,13 +75,6 @@ class BioformatsReader(Reader): see: https://docs.openmicroscopy.org/bio-formats/latest/formats/options.html For example: to turn off chunkmap table reading for ND2 files, use `options={"nativend2.chunkmap": False}` - dask_tiles: bool, optional - Whether to chunk the bioformats dask array by tiles to easily read sub-regions - with numpy-like array indexing - Defaults to false and iamges are read by entire planes - tile_size: Optional[Tuple[int, int]] - Tuple that sets the tile size of y and x axis, respectively - By default, it will use optimal values computed by bioformats itself Raises ------ exceptions.UnsupportedFileFormatError @@ -107,8 +100,6 @@ def __init__( original_meta: bool = False, memoize: Union[int, bool] = 0, options: Dict[str, bool] = {}, - dask_tiles: bool = False, - tile_size: Optional[Tuple[int, int]] = None, ): self._fs, self._path = io_utils.pathlike_to_fs(image, enforce_exists=True) # Catch non-local file system @@ -122,8 +113,6 @@ def __init__( "options": options, "original_meta": original_meta, "memoize": memoize, - "dask_tiles": dask_tiles, - "tile_size": tile_size, } try: with BioFile(self._path, **self._bf_kwargs) as rdr: # type: ignore @@ -174,8 +163,59 @@ def physical_pixel_sizes(self) -> types.PhysicalPixelSizes: ) def _to_xarray(self, delayed: bool = True) -> xr.DataArray: + + with BioFile(self._path, **self._bf_kwargs) as rdr: # type: ignore + rdr.set_series(self.current_scene_index) + + dims = ( + dimensions.DEFAULT_DIMENSION_ORDER_WITH_MOSAIC_TILES_AND_SAMPLES + if rdr.core_meta.is_rgb + else dimensions.DEFAULT_DIMENSION_ORDER_WITH_MOSAIC_TILES + ) + + image_data = ( + rdr.to_dask(self.current_scene_index) if delayed else rdr.to_numpy() + ) + image_data = image_data.reshape((1,) + image_data.shape) + + _, coords = metadata_utils.get_dims_and_coords_from_ome( + ome=rdr.ome_metadata, + scene_index=self.current_scene_index, + ) + + return xr.DataArray( + image_data, + dims=list(dims), + coords=coords, # type: ignore + attrs={ + constants.METADATA_UNPROCESSED: rdr.ome_xml, + constants.METADATA_PROCESSED: rdr.ome_metadata, + }, + ) + + def _construct_mosaic_xarray( + self, + delayed: bool = True, + tile_size: Optional[Tuple[int, int]] = None, + ) -> xr.DataArray: + with BioFile(self._path, **self._bf_kwargs) as rdr: # type: ignore - image_data = rdr.to_dask() if delayed else rdr.to_numpy() + rdr.set_series(self.current_scene_index) + + dims = ( + dimensions.DEFAULT_DIMENSION_ORDER_WITH_SAMPLES + if rdr.core_meta.is_rgb + else dimensions.DEFAULT_DIMENSION_ORDER + ) + + image_data = ( + rdr.to_dask( + self.current_scene_index, dask_tiles=True, tile_size=tile_size + ) + if delayed + else rdr.to_numpy() + ) + _, coords = metadata_utils.get_dims_and_coords_from_ome( ome=rdr.ome_metadata, scene_index=self.current_scene_index, @@ -183,9 +223,7 @@ def _to_xarray(self, delayed: bool = True) -> xr.DataArray: return xr.DataArray( image_data, - dims=dimensions.DEFAULT_DIMENSION_ORDER_LIST_WITH_SAMPLES - if rdr.core_meta.is_rgb - else dimensions.DEFAULT_DIMENSION_ORDER_LIST, + dims=list(dims), coords=coords, # type: ignore attrs={ constants.METADATA_UNPROCESSED: rdr.ome_xml, @@ -193,6 +231,12 @@ def _to_xarray(self, delayed: bool = True) -> xr.DataArray: }, ) + def _get_stitched_dask_mosaic(self) -> xr.DataArray: + return self._construct_mosaic_xarray(delayed=True) + + def _get_stitched_mosaic(self) -> xr.DataArray: + return self._construct_mosaic_xarray(delayed=False) + @staticmethod def bioformats_version() -> str: """The version of the bioformats_package.jar being used.""" @@ -309,16 +353,6 @@ def __init__( self._lock = Lock() self.set_series(series) - self.dask_tiles = dask_tiles - if self.dask_tiles: - if tile_size is None: - self.tile_size = ( - self._r.getOptimalTileHeight(), - self._r.getOptimalTileWidth(), - ) - else: - self.tile_size = tile_size - def set_series(self, series: int = 0) -> None: self._r.setSeries(series) self._core_meta = CoreMeta( @@ -367,9 +401,12 @@ def to_numpy(self, series: Optional[int] = None) -> np.ndarray: """ return np.asarray(self.to_dask(series)) - def to_dask(self, series: Optional[int] = None, - dask_tiles: bool = False, - tile_size: Optional[Tuple[int,int]] = None,) -> DaskArrayProxy: + def to_dask( + self, + series: Optional[int] = None, + dask_tiles: bool = False, + tile_size: Optional[Tuple[int, int]] = None, + ) -> DaskArrayProxy: """Create dask array for the specified or current series. Note: the order of the returned array will *always* be `TCZYX[r]`, @@ -411,13 +448,18 @@ def to_dask(self, series: Optional[int] = None, else: yx_tile_size = tile_size chunks = _get_dask_tile_chunks(nt, nc, nz, ny, nx, yx_tile_size) + dask_chunk_func = lambda block_id: self._dask_chunk( + block_id, dask_tiles=dask_tiles, tile_size=yx_tile_size + ) else: chunks = ((1,) * nt, (1,) * nc, (1,) * nz, ny, nx) + dask_chunk_func = self._dask_chunk if nrgb > 1: chunks = chunks + (nrgb,) # type: ignore + arr = da.map_blocks( - self._dask_chunk, + dask_chunk_func, chunks=chunks, dtype=self.core_meta.dtype, ) @@ -518,7 +560,12 @@ def _get_plane( return im - def _dask_chunk(self, block_id: Tuple[int, ...]) -> np.ndarray: + def _dask_chunk( + self, + block_id: Tuple[int, ...], + dask_tiles: bool = False, + tile_size: Optional[Tuple[int, int]] = None, + ) -> np.ndarray: """Retrieve `block_id` from array. This function is for map_blocks (called in `to_dask`). @@ -529,10 +576,10 @@ def _dask_chunk(self, block_id: Tuple[int, ...]) -> np.ndarray: # block_id will be coming in as (T, C, Z, Y, X). t, c, z, y, x, *_ = block_id - if self.dask_tiles: + if dask_tiles: *_, ny, nx, _ = self.core_meta.shape - y_slice = _axis_id_to_slice(y, self.tile_size[0], ny) - x_slice = _axis_id_to_slice(x, self.tile_size[1], nx) + y_slice = _axis_id_to_slice(y, tile_size[0], ny) + x_slice = _axis_id_to_slice(x, tile_size[1], nx) im = self._get_plane(t, c, z, y_slice, x_slice) else: im = self._get_plane(t, c, z) From 9e8e102139da55c0bdeac4e5896f224fda189e9c Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Mon, 18 Oct 2021 13:52:17 -0500 Subject: [PATCH 06/14] drop mosaic axis if == 1 after processing --- aicsimageio/aics_image.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index 41d028b75..e87bcc4e6 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -398,6 +398,12 @@ def xarray_dask_data(self) -> xr.DataArray: ) ) + if "M" in self._xarray_dask_data.dims: + m_idx = self._xarray_dask_data.dims.index("M") + if self._xarray_dask_data.shape[m_idx] == 1: + self._xarray_dask_data = np.squeeze( + self._xarray_dask_data, axis=m_idx + ) return self._xarray_dask_data @property From 7626361c0b5497870aa98fe44350a23464edcb21 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Tue, 26 Oct 2021 10:37:19 -0500 Subject: [PATCH 07/14] Revert to argument for `dask_tiles` --- aicsimageio/readers/bioformats_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index 1a7aedd09..1cfe8eda5 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -276,7 +276,7 @@ def __init__( memoize: Union[int, bool] = 0, options: Dict[str, bool] = {}, dask_tiles: bool = False, - tile_size: int = None, + tile_size: Optional[Tuple[int, int]] = None, ): try: loci = get_loci() From 61efe4d2c0a649a03f16d27087eec13d3fa1120e Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Tue, 26 Oct 2021 10:38:02 -0500 Subject: [PATCH 08/14] add tests for using `dask_tiles` and `tile_size` in `BioformatsReader` --- .../tests/readers/test_bioformats_reader.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/aicsimageio/tests/readers/test_bioformats_reader.py b/aicsimageio/tests/readers/test_bioformats_reader.py index fa65117dd..804544301 100644 --- a/aicsimageio/tests/readers/test_bioformats_reader.py +++ b/aicsimageio/tests/readers/test_bioformats_reader.py @@ -404,3 +404,54 @@ def test_multi_scene_bioformats_reader( second_scene_shape=second_scene_shape, second_scene_dtype=np.dtype(np.uint16), ) + + +@pytest.mark.parametrize( + "filename, ", + [ + ("CMU-1-Small-Region.svs"), + ], +) +def test_bioformats_dask_tiling_shapes(filename: str) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + bf_tiled = BioformatsReader(uri, dask_tiles=True) + bf_fullplane = BioformatsReader(uri, dask_tiles=False) + bf_tiled_set = BioformatsReader(uri, dask_tiles=True, tile_size=(1024, 1024)) + + np.testing.assert_array_equal( + bf_tiled.dask_data.shape, bf_fullplane.dask_data.shape + ) + np.testing.assert_array_equal( + bf_tiled_set.dask_data.shape, bf_fullplane.dask_data.shape + ) + np.testing.assert_array_equal( + bf_tiled.dask_data.shape, bf_fullplane.dask_data.shape + ) + np.testing.assert_array_equal(bf_tiled.dask_data.chunksize, (1, 1, 1, 240, 240, 3)) + np.testing.assert_array_equal( + bf_tiled_set.dask_data.chunksize, (1, 1, 1, 1024, 1024, 3) + ) + + +@pytest.mark.parametrize( + "filename, ", + [("s_1_t_1_c_10_z_1.ome.tiff"), ("CMU-1-Small-Region.svs")], +) +def test_bioformats_dask_tiling_read(filename: str) -> None: + # Construct full filepath + uri = get_resource_full_path(filename, LOCAL) + + # Run checks + bf_tiled = BioformatsReader(uri, dask_tiles=True) + bf_tiled_set = BioformatsReader(uri, dask_tiles=True, tile_size=(128, 128)) + bf_fullplane = BioformatsReader(uri, dask_tiles=False) + + arr_tiled = bf_tiled.dask_data.compute() + arr_fullplane = bf_fullplane.dask_data.compute() + arr_tiled_set = bf_tiled_set.dask_data.compute() + + np.testing.assert_array_equal(arr_tiled, arr_fullplane) + np.testing.assert_array_equal(arr_tiled_set, arr_fullplane) From ca9dfebffdf73a98b20b2789f44f13c47a3c8009 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Tue, 26 Oct 2021 10:43:47 -0500 Subject: [PATCH 09/14] Remove `M` check in `AICSImage` --- aicsimageio/aics_image.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index e87bcc4e6..41d028b75 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -398,12 +398,6 @@ def xarray_dask_data(self) -> xr.DataArray: ) ) - if "M" in self._xarray_dask_data.dims: - m_idx = self._xarray_dask_data.dims.index("M") - if self._xarray_dask_data.shape[m_idx] == 1: - self._xarray_dask_data = np.squeeze( - self._xarray_dask_data, axis=m_idx - ) return self._xarray_dask_data @property From 3311a2d89e99ad9a027e91b02e5d6d3d7d7e9efc Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Tue, 26 Oct 2021 10:55:00 -0500 Subject: [PATCH 10/14] Type checking for added functions --- aicsimageio/readers/bioformats_reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index 1cfe8eda5..ef512ab7a 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -561,13 +561,15 @@ def _pixtype2dtype(pixeltype: int, little_endian: bool) -> np.dtype: def _get_dask_tile_chunks( nt: int, nc: int, nz: int, ny: int, nx: int, tile_size: Tuple[int, int] -) -> Tuple[Tuple, Tuple, Tuple, Tuple, Tuple]: +) -> Tuple[ + Tuple[int, ...], Tuple[int, ...], Tuple[int, ...], Tuple[int, ...], Tuple[int, ...] +]: """Returns chunking tuples (length of each chunk in each axis) after tiling. I.e., if nx == 2048 and tile_size == 1024, chunks for x axis will be (1024,1024)""" y_tile_size, x_tile_size = tile_size - def _chunk_by_tile_size(n_px, tile_length): + def _chunk_by_tile_size(n_px: int, tile_length: int) -> Tuple[int, ...]: n_splits = n_px / tile_length n_full_tiles = np.floor(n_splits) From b5544ce7f14ec6431f374f21ee26176b6f5166e9 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Tue, 26 Oct 2021 10:56:52 -0500 Subject: [PATCH 11/14] Type checking for added functions --- aicsimageio/readers/bioformats_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index ef512ab7a..9ce5538bd 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -400,7 +400,7 @@ def to_dask(self, series: Optional[int] = None) -> DaskArrayProxy: if self.dask_tiles: chunks = _get_dask_tile_chunks(nt, nc, nz, ny, nx, self.tile_size) else: - chunks = ((1,) * nt, (1,) * nc, (1,) * nz, ny, nx) + chunks = ((1,) * nt, (1,) * nc, (1,) * nz, (ny,), (nx,)) if nrgb > 1: chunks = chunks + (nrgb,) # type: ignore From 459ec559f74812b5a214eebf6896e10b474aa7fd Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Tue, 26 Oct 2021 14:26:21 -0500 Subject: [PATCH 12/14] move `_chunk...` function out of other function --- aicsimageio/readers/bioformats_reader.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index 9ce5538bd..04104697c 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -559,6 +559,18 @@ def _pixtype2dtype(pixeltype: int, little_endian: bool) -> np.dtype: return np.dtype(("<" if little_endian else ">") + fmt2type[pixeltype]) +def _chunk_by_tile_size(n_px: int, tile_length: int) -> Tuple[int, ...]: + n_splits = n_px / tile_length + n_full_tiles = np.floor(n_splits) + + if n_splits.is_integer(): + tile_chunks = (int(tile_length),) * int(n_full_tiles) + else: + edge_tile = n_px - (n_full_tiles * tile_length) + tile_chunks = (int(tile_length),) * int(n_full_tiles) + (int(edge_tile),) + return tile_chunks + + def _get_dask_tile_chunks( nt: int, nc: int, nz: int, ny: int, nx: int, tile_size: Tuple[int, int] ) -> Tuple[ @@ -569,17 +581,6 @@ def _get_dask_tile_chunks( y_tile_size, x_tile_size = tile_size - def _chunk_by_tile_size(n_px: int, tile_length: int) -> Tuple[int, ...]: - n_splits = n_px / tile_length - n_full_tiles = np.floor(n_splits) - - if n_splits.is_integer(): - tile_chunks = (int(tile_length),) * int(n_full_tiles) - else: - edge_tile = n_px - (n_full_tiles * tile_length) - tile_chunks = (int(tile_length),) * int(n_full_tiles) + (int(edge_tile),) - return tile_chunks - y_tiling_chunks = _chunk_by_tile_size(ny, y_tile_size) x_tiling_chunks = _chunk_by_tile_size(nx, x_tile_size) From c04ab9d1933a7848779dfd43c67ad3d6a5eb8f72 Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Wed, 27 Oct 2021 07:06:20 -0500 Subject: [PATCH 13/14] update test hash --- scripts/TEST_RESOURCES_HASH.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/TEST_RESOURCES_HASH.txt b/scripts/TEST_RESOURCES_HASH.txt index f892f4b56..f69263715 100644 --- a/scripts/TEST_RESOURCES_HASH.txt +++ b/scripts/TEST_RESOURCES_HASH.txt @@ -1 +1 @@ -f65e7b1ecd3eccf384e2fb1333fff841bbde06a1d38b9985a613ef06dd7530bb +ac6e65ba11f525fe15910b916e97332acc9fddb6353cdfabae78fc50938edbe4 From cbc9306045eb914b5057dc3936e9661720f7a11b Mon Sep 17 00:00:00 2001 From: Heath Patterson Date: Thu, 4 Nov 2021 08:08:51 -0500 Subject: [PATCH 14/14] Update test hash --- scripts/TEST_RESOURCES_HASH.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/TEST_RESOURCES_HASH.txt b/scripts/TEST_RESOURCES_HASH.txt index f69263715..e7a7918e3 100644 --- a/scripts/TEST_RESOURCES_HASH.txt +++ b/scripts/TEST_RESOURCES_HASH.txt @@ -1 +1 @@ -ac6e65ba11f525fe15910b916e97332acc9fddb6353cdfabae78fc50938edbe4 +3a475850cd52c29f4cc047289e770434fe5fe33863742fa03e08623635bf49bd