From c64aca92b41d5420a7ce54314443a0378802520d Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 11 Aug 2021 15:02:13 -0700 Subject: [PATCH 01/29] Initital glob reader GlobReader WIP glob reader with working _read_delayed working _read_delayed and _read_immediate for 2d tiffs better check for single file shape formatting working 2d and 3d read immediate docstring and formatting small cleanup --- aicsimageio/readers/__init__.py | 3 + aicsimageio/readers/glob_reader.py | 354 +++++++++++++++++++++++++++++ 2 files changed, 357 insertions(+) create mode 100644 aicsimageio/readers/glob_reader.py diff --git a/aicsimageio/readers/__init__.py b/aicsimageio/readers/__init__.py index f76022771..f70796940 100644 --- a/aicsimageio/readers/__init__.py +++ b/aicsimageio/readers/__init__.py @@ -15,6 +15,7 @@ from .ome_tiff_reader import OmeTiffReader # noqa: F401 from .reader import Reader from .tiff_reader import TiffReader # noqa: F401 + from .glob_reader import GlobReader # noqa: F401 # add ".relativepath.ClassName" @@ -27,6 +28,7 @@ ".nd2_reader.ND2Reader", ".ome_tiff_reader.OmeTiffReader", ".tiff_reader.TiffReader", + ".glob_reader.GlobReader", ) _LOOKUP = {k.rsplit(".", 1)[-1]: k for k in _READERS} __all__ = list(_LOOKUP) @@ -40,3 +42,4 @@ def __getattr__(name: str) -> Type["Reader"]: mod = import_module(path, __name__) return getattr(mod, clsname) raise AttributeError(f"module {__name__!r} has no attribute import name {name!r}") + diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py new file mode 100644 index 000000000..1a730949e --- /dev/null +++ b/aicsimageio/readers/glob_reader.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from typing import Any, Dict, List, Optional, Tuple, Union, Callable + +import re +import glob +import dask.array as da +import numpy as np +import pandas as pd +import xarray as xr +from dask import delayed +from fsspec.spec import AbstractFileSystem +from tifffile import TiffFile, TiffFileError, imread +from tifffile.tifffile import TiffTags + +from .. import constants, exceptions, types +from ..dimensions import ( + DEFAULT_CHUNK_DIMS, + REQUIRED_CHUNK_DIMS, + DimensionNames, + DEFAULT_DIMENSION_ORDER_LIST_WITH_SAMPLES, +) +from ..metadata import utils as metadata_utils +from ..utils import io_utils +from .reader import Reader + + +class GlobReader(Reader): + + """ + Wraps the tifffile imread API to provide the same aicsimageio Reader API but for + multifile tiff datasets (and other tifffile supported) images. + + Parameters + ---------- + glob_in: Union[str, List[str]] + Glob string that identifies all files to be loaded or a list + of paths to the files as returned by glob. + indexer: Union[Callable, pandas.DataFrame] + If callable, should consume each filename and return a pd.Series with index + corresponding to the dimensions and values corresponding to the index + chunk_dims: Union[str, List[str]] + Which dimensions to create chunks for. + Default: DEFAULT_CHUNK_DIMS + Note: Dimensions.SpatialY, Dimensions.SpatialX, and DimensionNames.Samples, + will always be added to the list if not present during dask array + construction. + dim_order: Optional[Union[List[str], str]] + A string of dimensions to be applied to all array(s) or a + list of string dimension names to be mapped onto the list of arrays + provided to image. I.E. "TYX". + Default: None (guess dimensions for single array or multiple arrays) + channel_names: Optional[Union[List[str], List[List[str]]]] + A list of string channel names to be applied to all array(s) or a + list of lists of string channel names to be mapped onto the list of arrays + provided to image. + Default: None (create OME channel IDs for names for single or multiple arrays) + single_file_shape : Optional[Tuple] + Expected shape for a single file of the set. If not provided, will attempt to + determine the shape from the first file found in the glob. + Default : None + single_file_dims : Optional[Tuple] + Dimensions that correspond to the data dimensions of a single file in the glob. + Default : ('Y', 'X') + """ + + @staticmethod + def _is_supported_image(fs: AbstractFileSystem, path: str, **kwargs: Any) -> bool: + try: + with fs.open(path) as open_resource: + with TiffFile(open_resource): + return True + + except (TiffFileError, TypeError): + return False + + def __init__( + self, + glob_in: Union[str, List[str]], + indexer: Union[pd.DataFrame, Callable] = None, + chunk_dims: Union[str, List[str]] = DEFAULT_CHUNK_DIMS, + dim_order: Optional[Union[List[str], str]] = None, + channel_names: Optional[Union[List[str], List[List[str]]]] = None, + single_file_shape: Optional[Tuple] = None, + single_file_dims: Optional[Tuple] = ( + DimensionNames.SpatialY, + DimensionNames.SpatialX, + ), + **kwargs: Any, + ): + + # Assmble glob list if given a string + if isinstance(glob_in, str): + file_series = pd.Series( + glob.glob(glob_in) + ) # pd.DataFrame({"filename":glob.glob(glob_in)}) + elif isinstance(glob_in, list): + file_series = pd.Series(glob_in) # pd.DataFrame({"filename": glob_in}) + + if len(file_series) == 0: + raise ValueError("No files found matching glob pattern") + + if indexer is None: + series_idx = [ + DimensionNames.Samples, + DimensionNames.Time, + DimensionNames.Channel, + DimensionNames.SpatialZ, + ] + indexer = lambda x: pd.Series( + re.findall(r"\d+", x.split("/")[-1]), index=series_idx + ).astype(int) + + if callable(indexer): + self._all_files = file_series.apply(indexer) + self._all_files["filename"] = file_series + elif isinstance(indexer, pd.DataFrame): + self._all_files = indexer + self._all_files["filename"] = file_series + + sort_order = [] + for dim in DEFAULT_DIMENSION_ORDER_LIST_WITH_SAMPLES: + if dim not in self._all_files.columns and dim not in single_file_dims: + self._all_files[dim] = 0 + if dim in self._all_files.columns: + sort_order.append(dim) + + self._all_files = self._all_files.sort_values(sort_order).reset_index(drop=True) + + # run tests on a single file (?) + self._fs, self._path = io_utils.pathlike_to_fs(self._all_files.iloc[0].filename) + + # Store params + if isinstance(chunk_dims, str): + chunk_dims = list(chunk_dims) + + # Run basic checks on dims and channel names + if isinstance(dim_order, list): + if len(dim_order) != len(self.scenes): + raise exceptions.ConflictingArgumentsError( + f"Number of dimension strings provided does not match the " + f"number of scenes found in the file. " + f"Number of scenes: {len(self.scenes)}, " + f"Number of provided dimension order strings: {len(dim_order)}" + ) + + # If provided a list + if isinstance(channel_names, list): + # If provided a list of lists + if len(channel_names) > 0 and isinstance(channel_names[0], list): + # Ensure that the outer list is the number of scenes + if len(channel_names) != len(self.scenes): + raise exceptions.ConflictingArgumentsError( + f"Number of channel name lists provided does not match the " + f"number of scenes found in the file. " + f"Number of scenes: {len(self.scenes)}, " + f"Provided channel name lists: {dim_order}" + ) + self._channel_names = channel_names + + self.chunk_dims = chunk_dims + + for dim in REQUIRED_CHUNK_DIMS: + if dim not in self.chunk_dims: + self.chunk_dims.append(dim) + + # Safety measure / "feature" + self.chunk_dims = [d.upper() for d in self.chunk_dims] + + self._dim_order = dim_order + self._channel_names = channel_names + + if single_file_shape is None: + with self._fs.open(self._path) as open_resource: + with TiffFile(open_resource) as tiff: + if tiff.is_shaped: + self._single_file_shape = tuple( + tiff.shaped_metadata[0]["shape"] + ) + else: + self._single_file_shape = tiff.pages[0].shape + + else: + self._single_file_shape = single_file_shape + + if len(single_file_dims) != len(self._single_file_shape): + raise exceptions.ConflictingArgumentsError( + f"Number of single file dimensions does not match the" + f"number of dimensions in a test file. " + f"Number of dimensions in file: {len(self._single_file_shape)}, " + f"Provided number of dimensions: {len(single_file_dims)}." + ) + + else: + self._single_file_dims = list(single_file_dims) + + # Enforce valid image + if not self._is_supported_image(self._fs, self._path): + raise exceptions.UnsupportedFileFormatError( + self.__class__.__name__, self._path + ) + + @property + def scenes(self) -> Tuple[str, ...]: + if self._scenes is None: + self._scenes = tuple( + metadata_utils.generate_ome_image_id(s) + for s in range(self._all_files[DimensionNames.Samples].nunique()) + ) + return self._scenes + + def _read_delayed(self) -> xr.DataArray: + + scene_files = self._all_files.loc[ + self._all_files[DimensionNames.Samples] == self.current_scene_index + ] + scene_files = scene_files.drop(DimensionNames.Samples, axis=1) + group_dims = [ + x for x in scene_files.columns if x not in self.chunk_dims + ["filename"] + ] + # cutoffs = .nunique().min().drop('filename') + # df = df.loc[(df.loc[:, ~df.columns.isin(['S', 'filename'])] < cutoffs).all('columns')] + chunks = np.zeros(scene_files[group_dims].nunique().values, dtype="object") + for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): + zarr_im = imread(val.filename.tolist(), aszarr=True) + darr = da.from_zarr(zarr_im).rechunk(-1) + if i == 0: + shape = ( + tuple( + x + for i, x in scene_files.nunique().iteritems() + if i in self.chunk_dims + ) + + self._single_file_shape + ) # darr.shape[-2:] + ordered_chunk_dims = [ + d for d in scene_files.columns if d in self.chunk_dims + ] + ordered_chunk_dims += [ + d + for d in self.chunk_dims + if d not in ordered_chunk_dims + [DimensionNames.Samples] + ] + darr = darr.reshape(shape) + chunks[idx] = darr + + chunks = np.expand_dims( + chunks, tuple(range(-1, -len(ordered_chunk_dims) - 1, -1)) + ) + + d_data = da.block(chunks.tolist()) + + dims = group_dims + ordered_chunk_dims + channel_names = self._get_channel_names_for_scene(dims) + + coords = self._get_coords( + dims, d_data.shape, self.current_scene_index, channel_names + ) + x_data = xr.DataArray(d_data, dims=dims, coords=coords) + + return x_data + + def _read_immediate(self) -> xr.DataArray: + scene = self.current_scene_index + + scene_files = self._all_files.loc[ + self._all_files[DimensionNames.Samples] == scene + ] + scene_files = scene_files.drop(DimensionNames.Samples, axis=1) + + arr = imread(scene_files.filename.tolist()) + + shape = ( + tuple(x for i, x in scene_files.nunique().drop("filename").iteritems()) + + self._single_file_shape + ) + arr = arr.reshape(shape) + + dims = scene_files.columns.drop("filename").values.tolist() + list( + self._single_file_dims + ) + channel_names = self._get_channel_names_for_scene(dims) + + coords = self._get_coords(dims, shape, scene, channel_names) + + x_data = xr.DataArray( + arr, + dims=dims, + coords=coords, + ) + + return x_data + + def _get_channel_names_for_scene(self, dims: List[str]) -> Optional[List[str]]: + # Fast return in None case + if self._channel_names is None: + return None + + # If channels was provided as a list of lists + if isinstance(self._channel_names[0], list): + scene_channels = self._channel_names[self.current_scene_index] + elif all(isinstance(c, str) for c in self._channel_names): + scene_channels = self._channel_names # type: ignore + else: + return None + + # If scene channels isn't None and no channel dimension raise error + if DimensionNames.Channel not in dims: + raise exceptions.ConflictingArgumentsError( + f"Provided channel names for scene with no channel dimension. " + f"Scene dims: {dims}, " + f"Provided channel names: {scene_channels}" + ) + + # If scene channels isn't the same length as the size of channel dim + if len(scene_channels) != image_shape[dims.index(DimensionNames.Channel)]: + raise exceptions.ConflictingArgumentsError( + f"Number of channel names provided does not match the " + f"size of the channel dimension for this scene. " + f"Scene shape: {image_shape}, " + f"Dims: {dims}, " + f"Provided channel names: {self._channel_names}", + ) + + return scene_channels # type: ignore + + @staticmethod + def _get_coords( + dims: List[str], + shape: Tuple[int, ...], + scene_index: int, + channel_names: Optional[List[str]], + ) -> Dict[str, Any]: + # Use dims for coord determination + coords: Dict[str, Any] = {} + + if channel_names is None: + # Get ImageId for channel naming + image_id = metadata_utils.generate_ome_image_id(scene_index) + + # Use range for channel indices + if DimensionNames.Channel in dims: + coords[DimensionNames.Channel] = [ + metadata_utils.generate_ome_channel_id( + image_id=image_id, channel_id=i + ) + for i in range(shape[dims.index(DimensionNames.Channel)]) + ] + else: + coords[DimensionNames.Channel] = channel_names + + return coords + From d031a0fc7e4113c1c25468cf29f09104822e95e3 Mon Sep 17 00:00:00 2001 From: John Russell Date: Mon, 30 Aug 2021 12:27:25 -0400 Subject: [PATCH 02/29] test glob reader on single frame tiffs --- aicsimageio/tests/readers/test_glob_reader.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 aicsimageio/tests/readers/test_glob_reader.py diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py new file mode 100644 index 000000000..700d7145f --- /dev/null +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -0,0 +1,29 @@ +#!/usr/bin/env/python + + +import os +import numpy as np +import xarray as xr +import tifffile as tiff + + +DATA_SHAPE = (2,3,4,5,6,7) # STCZYX + +def test_glob_reader(): + data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) + try: + os.mkdir("fake_data") + except FileExistsError: + pass + + for s in range(DATA_SHAPE[0]): + for t in range(DATA_SHAPE[1]): + for c in range(DATA_SHAPE[2]): + for z in range(DATA_SHAPE[3]): + im = data[s,t,c,z] + tiff.imsave(f"fake_data/S{s}_T{t}_C{c}_Z{z}.tif", im, dtype=np.uint16) + + + dims = list("STCZYX") + coords = {'C':[f'Channel:0:{i}' for i in range(DATA_SHAPE[2])]} + x_data = xr.DataArray(data, dims=dims, coords=coords) From 3050f24977d6f38840d2e7eb672a660743497f70 Mon Sep 17 00:00:00 2001 From: John Russell Date: Mon, 30 Aug 2021 12:31:09 -0400 Subject: [PATCH 03/29] correct file this time --- aicsimageio/tests/readers/test_glob_reader.py | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index 700d7145f..7c13d7f5e 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -1,29 +1,42 @@ -#!/usr/bin/env/python - - +#! usr/env/bin/python import os import numpy as np import xarray as xr import tifffile as tiff +import aicsimageio +from pathlib import Path - +TIFF_3D = False DATA_SHAPE = (2,3,4,5,6,7) # STCZYX -def test_glob_reader(): + +def make_fake_data_2d(path): + data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) - try: - os.mkdir("fake_data") - except FileExistsError: - pass + dims = list("STCZYX") + image_ids = [aicsimageio.metadata.utils.generate_ome_image_id(i) for i in range(DATA_SHAPE[0])] + + x_data = xr.DataArray(data, dims=dims)#, coords=coords) + for s in range(DATA_SHAPE[0]): for t in range(DATA_SHAPE[1]): for c in range(DATA_SHAPE[2]): - for z in range(DATA_SHAPE[3]): - im = data[s,t,c,z] - tiff.imsave(f"fake_data/S{s}_T{t}_C{c}_Z{z}.tif", im, dtype=np.uint16) - + for z in range(DATA_SHAPE[3]): + im = data[s,t,c,z] + tiff.imsave(str(path/f"S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16) + return x_data - dims = list("STCZYX") - coords = {'C':[f'Channel:0:{i}' for i in range(DATA_SHAPE[2])]} - x_data = xr.DataArray(data, dims=dims, coords=coords) +def test_glob_reader_2d(tmp_path: Path): + reference = make_fake_data_2d(tmp_path) + gr = aicsimageio.readers.GlobReader(str(tmp_path/"*.tif")) + + for i, s in enumerate(gr.scenes): + gr.set_scene(s) + assert np.all(reference.isel(S=i).data == gr.xarray_dask_data.data) + assert np.all(reference.isel(S=i).data == gr.xarray_data.data) + + + +# im = data[s,t,c] +# tiff.imsave(f"fake_data/S{s}_T{t}_C{c}.tif", im, dtype=np.uint16) From 6b3b1dd45aac3cf70d28a6f31bef3a7fa69e2a0b Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 1 Sep 2021 18:01:55 -0400 Subject: [PATCH 04/29] guess glob reader for list of str or str with wildcard --- aicsimageio/aics_image.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index 41d028b75..8e94131fe 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -15,6 +15,7 @@ from .metadata import utils as metadata_utils from .readers.reader import Reader from .types import PhysicalPixelSizes +from .readers import GlobReader ############################################################################### @@ -148,8 +149,15 @@ def determine_reader(image: types.ImageLike, **kwargs: Any) -> Type[Reader]: exceptions.UnsupportedFileFormatError No reader could be found that supports the provided image. """ + + if isinstance(image, list) and isinstance(image[0], str): + return GlobReader + elif isinstance(image, str) and "*" in image: + return GlobReader + # Try reader detection based off of file path extension if isinstance(image, (str, Path)): + path = str(image) # Check for extension in FORMAT_IMPLEMENTATIONS From 8ca9c3685a4f0a7e5601e48df078f82940e8e53f Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 1 Sep 2021 18:03:30 -0400 Subject: [PATCH 05/29] add list of str as valid image type --- aicsimageio/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aicsimageio/types.py b/aicsimageio/types.py index 08290470b..de313eb66 100644 --- a/aicsimageio/types.py +++ b/aicsimageio/types.py @@ -14,7 +14,7 @@ PathLike = Union[str, Path] ArrayLike = Union[np.ndarray, da.Array] MetaArrayLike = Union[ArrayLike, xr.DataArray] -ImageLike = Union[PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike]] +ImageLike = Union[PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike], List[str]] # Image Utility Types From fa18e6c3da06069774153d0311793de25fa2b824 Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 1 Sep 2021 18:36:27 -0400 Subject: [PATCH 06/29] use pathlib for filename in default indexer --- aicsimageio/readers/glob_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 1a730949e..11a4797b8 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -5,6 +5,7 @@ import re import glob +from pathlib import Path import dask.array as da import numpy as np import pandas as pd @@ -109,7 +110,7 @@ def __init__( DimensionNames.SpatialZ, ] indexer = lambda x: pd.Series( - re.findall(r"\d+", x.split("/")[-1]), index=series_idx + re.findall(r"\d+", Path(x).name), index=series_idx ).astype(int) if callable(indexer): From 8983f70d8be6d03ac4e925f5749b007d569e6ab8 Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 1 Sep 2021 20:58:11 -0400 Subject: [PATCH 07/29] handle stacking along dimensions included in single files --- aicsimageio/readers/glob_reader.py | 50 ++++++++++++++++-------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 11a4797b8..aa88fa5fd 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -95,7 +95,7 @@ def __init__( if isinstance(glob_in, str): file_series = pd.Series( glob.glob(glob_in) - ) # pd.DataFrame({"filename":glob.glob(glob_in)}) + ) elif isinstance(glob_in, list): file_series = pd.Series(glob_in) # pd.DataFrame({"filename": glob_in}) @@ -179,9 +179,9 @@ def __init__( self._single_file_shape = tuple( tiff.shaped_metadata[0]["shape"] ) - else: - self._single_file_shape = tiff.pages[0].shape - + elif len(tiff.series)==1: + self._single_file_shape = tiff.series[0].shape + else: self._single_file_shape = single_file_shape @@ -196,6 +196,7 @@ def __init__( else: self._single_file_dims = list(single_file_dims) + self._single_file_sizes = dict(zip(self._single_file_dims, self._single_file_shape)) # Enforce valid image if not self._is_supported_image(self._fs, self._path): raise exceptions.UnsupportedFileFormatError( @@ -220,29 +221,32 @@ def _read_delayed(self) -> xr.DataArray: group_dims = [ x for x in scene_files.columns if x not in self.chunk_dims + ["filename"] ] - # cutoffs = .nunique().min().drop('filename') - # df = df.loc[(df.loc[:, ~df.columns.isin(['S', 'filename'])] < cutoffs).all('columns')] + chunks = np.zeros(scene_files[group_dims].nunique().values, dtype="object") + + shape = () + for i,x in scene_files.nunique().iteritems(): + if i not in group_dims +['filename']: + if i not in self._single_file_dims: + shape += (x,) + else: + shape += (self._single_file_sizes[i]*x,) + + for i,x in self._single_file_sizes.items(): + if i not in scene_files.columns: + shape += (x,) + + ordered_chunk_dims = [ + d for d in scene_files.columns if d in self.chunk_dims + ] + ordered_chunk_dims += [ + d for d in self.chunk_dims + if d not in ordered_chunk_dims + [DimensionNames.Samples] + ] + for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): zarr_im = imread(val.filename.tolist(), aszarr=True) darr = da.from_zarr(zarr_im).rechunk(-1) - if i == 0: - shape = ( - tuple( - x - for i, x in scene_files.nunique().iteritems() - if i in self.chunk_dims - ) - + self._single_file_shape - ) # darr.shape[-2:] - ordered_chunk_dims = [ - d for d in scene_files.columns if d in self.chunk_dims - ] - ordered_chunk_dims += [ - d - for d in self.chunk_dims - if d not in ordered_chunk_dims + [DimensionNames.Samples] - ] darr = darr.reshape(shape) chunks[idx] = darr From 42c8aeda227a4cb4a4d24f318fdc8f0a8534bdc1 Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 1 Sep 2021 22:15:30 -0400 Subject: [PATCH 08/29] stopping for now. still working out all the cases of stacking multidim files --- aicsimageio/readers/glob_reader.py | 91 ++++++++++++++++++------------ 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index aa88fa5fd..10f4d5ca5 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -93,9 +93,7 @@ def __init__( # Assmble glob list if given a string if isinstance(glob_in, str): - file_series = pd.Series( - glob.glob(glob_in) - ) + file_series = pd.Series(glob.glob(glob_in)) elif isinstance(glob_in, list): file_series = pd.Series(glob_in) # pd.DataFrame({"filename": glob_in}) @@ -179,9 +177,9 @@ def __init__( self._single_file_shape = tuple( tiff.shaped_metadata[0]["shape"] ) - elif len(tiff.series)==1: + elif len(tiff.series) == 1: self._single_file_shape = tiff.series[0].shape - + else: self._single_file_shape = single_file_shape @@ -196,7 +194,9 @@ def __init__( else: self._single_file_dims = list(single_file_dims) - self._single_file_sizes = dict(zip(self._single_file_dims, self._single_file_shape)) + self._single_file_sizes = dict( + zip(self._single_file_dims, self._single_file_shape) + ) # Enforce valid image if not self._is_supported_image(self._fs, self._path): raise exceptions.UnsupportedFileFormatError( @@ -218,36 +218,36 @@ def _read_delayed(self) -> xr.DataArray: self._all_files[DimensionNames.Samples] == self.current_scene_index ] scene_files = scene_files.drop(DimensionNames.Samples, axis=1) + group_dims = [ x for x in scene_files.columns if x not in self.chunk_dims + ["filename"] ] - + chunks = np.zeros(scene_files[group_dims].nunique().values, dtype="object") - shape = () - for i,x in scene_files.nunique().iteritems(): - if i not in group_dims +['filename']: - if i not in self._single_file_dims: - shape += (x,) - else: - shape += (self._single_file_sizes[i]*x,) + chunk_shape = self._get_shape_for_scene(scene_files, group_dims) + + ordered_chunk_dims = [d for d in scene_files.columns if d in self.chunk_dims] - for i,x in self._single_file_sizes.items(): - if i not in scene_files.columns: - shape += (x,) - - ordered_chunk_dims = [ - d for d in scene_files.columns if d in self.chunk_dims - ] ordered_chunk_dims += [ - d for d in self.chunk_dims + d + for d in self.chunk_dims if d not in ordered_chunk_dims + [DimensionNames.Samples] ] + + #Need to allow rechunk in ways other than -1 + # so that when you want planes of a file in separate chunks e.g. files are ZYX and chunk_dims is TC + # you can still reshape the dask array but then rechunk appropriately + image_dims_not_in_chunk = [x for x in self._single_file_dims if x not in ordered_chunk_dims] + print(image_dims_not_in_chunk) + print(chunk_shape) + print(ordered_chunk_dims) for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): zarr_im = imread(val.filename.tolist(), aszarr=True) darr = da.from_zarr(zarr_im).rechunk(-1) - darr = darr.reshape(shape) + if i==0: print(darr.shape) + darr = darr.reshape(chunk_shape) chunks[idx] = darr chunks = np.expand_dims( @@ -263,40 +263,62 @@ def _read_delayed(self) -> xr.DataArray: dims, d_data.shape, self.current_scene_index, channel_names ) x_data = xr.DataArray(d_data, dims=dims, coords=coords) + + if self._dim_order is not None: + print(self._dim_order) + x_data.transpose(*list(self._dim_order)) return x_data def _read_immediate(self) -> xr.DataArray: - scene = self.current_scene_index - scene_files = self._all_files.loc[ - self._all_files[DimensionNames.Samples] == scene + self._all_files[DimensionNames.Samples] == self.current_scene_index ] scene_files = scene_files.drop(DimensionNames.Samples, axis=1) arr = imread(scene_files.filename.tolist()) - shape = ( - tuple(x for i, x in scene_files.nunique().drop("filename").iteritems()) - + self._single_file_shape - ) + shape = self._get_shape_for_scene(scene_files) + arr = arr.reshape(shape) - dims = scene_files.columns.drop("filename").values.tolist() + list( - self._single_file_dims - ) + dims = scene_files.columns.drop("filename").values.tolist() + file_dims = [x for x in self._single_file_dims if x not in dims] + dims += file_dims + channel_names = self._get_channel_names_for_scene(dims) - coords = self._get_coords(dims, shape, scene, channel_names) + coords = self._get_coords(dims, shape, self.current_scene_index, channel_names) x_data = xr.DataArray( arr, dims=dims, coords=coords, ) + if self._dim_order is not None: + print(self._dim_order) + x_data.transpose(*list(self._dim_order)) return x_data + def _get_shape_for_scene( + self, scene_files: pd.DataFrame, group_dims: Union[List[str], type(None)] = None + ): + if group_dims is None: + group_dims = [] + shape = () + for i, x in scene_files.nunique().iteritems(): + if i not in group_dims + ["filename"]: + if i not in self._single_file_dims: + shape += (x,) + else: + shape += (self._single_file_sizes[i] * x,) + + for i, x in self._single_file_sizes.items(): + if i not in scene_files.columns: + shape += (x,) + return shape + def _get_channel_names_for_scene(self, dims: List[str]) -> Optional[List[str]]: # Fast return in None case if self._channel_names is None: @@ -356,4 +378,3 @@ def _get_coords( coords[DimensionNames.Channel] = channel_names return coords - From 5f812881f6828fdb9de19764636ab95c42d245a9 Mon Sep 17 00:00:00 2001 From: John Russell Date: Thu, 2 Sep 2021 11:55:11 -0400 Subject: [PATCH 09/29] refactor to support globbing along dimenions contained in single files --- aicsimageio/readers/glob_reader.py | 107 +++++++++++++++-------------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 10f4d5ca5..38390e505 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -5,6 +5,7 @@ import re import glob +from collections import OrderedDict from pathlib import Path import dask.array as da import numpy as np @@ -218,77 +219,76 @@ def _read_delayed(self) -> xr.DataArray: self._all_files[DimensionNames.Samples] == self.current_scene_index ] scene_files = scene_files.drop(DimensionNames.Samples, axis=1) - + scene_nunique = scene_files.nunique() group_dims = [ x for x in scene_files.columns if x not in self.chunk_dims + ["filename"] ] - chunks = np.zeros(scene_files[group_dims].nunique().values, dtype="object") - - chunk_shape = self._get_shape_for_scene(scene_files, group_dims) - - ordered_chunk_dims = [d for d in scene_files.columns if d in self.chunk_dims] - - ordered_chunk_dims += [ - d - for d in self.chunk_dims - if d not in ordered_chunk_dims + [DimensionNames.Samples] - ] + group_sizes = OrderedDict([(d, scene_nunique[d]) for d in group_dims]) + chunk_sizes = self._get_sizes_for_scene(scene_nunique, group_dims) - #Need to allow rechunk in ways other than -1 - # so that when you want planes of a file in separate chunks e.g. files are ZYX and chunk_dims is TC - # you can still reshape the dask array but then rechunk appropriately - image_dims_not_in_chunk = [x for x in self._single_file_dims if x not in ordered_chunk_dims] - print(image_dims_not_in_chunk) - print(chunk_shape) - print(ordered_chunk_dims) - - for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): - zarr_im = imread(val.filename.tolist(), aszarr=True) - darr = da.from_zarr(zarr_im).rechunk(-1) - if i==0: print(darr.shape) - darr = darr.reshape(chunk_shape) - chunks[idx] = darr - - chunks = np.expand_dims( - chunks, tuple(range(-1, -len(ordered_chunk_dims) - 1, -1)) - ) + + # Assemble the dask array + if len(group_dims)>0: #use groupby to assemble array out of chunks + chunks = np.zeros(tuple(group_sizes.values()), dtype="object") + + for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): + zarr_im = imread(val.filename.tolist(), aszarr=True) + darr = da.from_zarr(zarr_im).rechunk(-1) + darr = darr.reshape(tuple(chunk_sizes.values())) + chunks[idx] = darr + + overlap_dims = group_sizes.keys() & chunk_sizes.keys() + expanded_shape = tuple(s for d,s in group_sizes.items() if d not in overlap_dims) + for d in chunk_sizes: + if d in overlap_dims: + expanded_shape += (group_sizes[d],) + else: + expanded_shape += (1,) + + chunks = chunks.reshape(expanded_shape) + d_data = da.block(chunks.tolist()) - d_data = da.block(chunks.tolist()) + else: # assemble array in a single chunk + zarr_im = imread(scene_files.filename.tolist(), aszarr=True) + darr = da.from_zarr(zarr_im).rechunk(-1) + d_data = darr.reshape(tuple(chunk_sizes.values())) - dims = group_dims + ordered_chunk_dims + # Assign dims and coords to construct xarray + dims = [d for d in group_dims if d not in overlap_dims] + list(chunk_sizes.keys()) channel_names = self._get_channel_names_for_scene(dims) coords = self._get_coords( dims, d_data.shape, self.current_scene_index, channel_names ) x_data = xr.DataArray(d_data, dims=dims, coords=coords) - if self._dim_order is not None: - print(self._dim_order) x_data.transpose(*list(self._dim_order)) return x_data def _read_immediate(self) -> xr.DataArray: + # Set up scene specific information scene_files = self._all_files.loc[ self._all_files[DimensionNames.Samples] == self.current_scene_index ] scene_files = scene_files.drop(DimensionNames.Samples, axis=1) + scene_nunique = scene_files.nunique() + scene_sizes = self._get_sizes_for_scene(scene_nunique) + + # Assemble array arr = imread(scene_files.filename.tolist()) - - shape = self._get_shape_for_scene(scene_files) - - arr = arr.reshape(shape) - + arr = arr.reshape(tuple(scene_sizes.values())) + + # Assign dims and coords to construct xarray dims = scene_files.columns.drop("filename").values.tolist() file_dims = [x for x in self._single_file_dims if x not in dims] dims += file_dims channel_names = self._get_channel_names_for_scene(dims) - coords = self._get_coords(dims, shape, self.current_scene_index, channel_names) + coords = self._get_coords(dims, arr.shape, self.current_scene_index, channel_names) x_data = xr.DataArray( arr, @@ -296,28 +296,31 @@ def _read_immediate(self) -> xr.DataArray: coords=coords, ) if self._dim_order is not None: - print(self._dim_order) x_data.transpose(*list(self._dim_order)) return x_data - def _get_shape_for_scene( - self, scene_files: pd.DataFrame, group_dims: Union[List[str], type(None)] = None + def _get_sizes_for_scene( + self, scene_files_nunique: pd.Series, group_dims: List[str] = [] ): - if group_dims is None: - group_dims = [] - shape = () - for i, x in scene_files.nunique().iteritems(): + + sizes = OrderedDict() + for i, x in scene_files_nunique.iteritems(): if i not in group_dims + ["filename"]: if i not in self._single_file_dims: - shape += (x,) + sizes[i] = x else: - shape += (self._single_file_sizes[i] * x,) + sizes[i] = self._single_file_sizes[i] * x + + for d,s in self._single_file_sizes.items(): + if d not in self.chunk_dims and d not in sizes: + sizes[d] = s for i, x in self._single_file_sizes.items(): - if i not in scene_files.columns: - shape += (x,) - return shape + if i not in scene_files_nunique.index: + sizes[i] = x + + return sizes def _get_channel_names_for_scene(self, dims: List[str]) -> Optional[List[str]]: # Fast return in None case From 562577439dfa3d58939bc31b9e2972a6cfa34ac6 Mon Sep 17 00:00:00 2001 From: John Russell Date: Thu, 2 Sep 2021 14:09:38 -0400 Subject: [PATCH 10/29] more fixes toward supporting multidim images --- aicsimageio/readers/glob_reader.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 38390e505..a0e160d89 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -21,6 +21,7 @@ DEFAULT_CHUNK_DIMS, REQUIRED_CHUNK_DIMS, DimensionNames, + DEFAULT_DIMENSION_ORDER, DEFAULT_DIMENSION_ORDER_LIST_WITH_SAMPLES, ) from ..metadata import utils as metadata_utils @@ -167,8 +168,12 @@ def __init__( # Safety measure / "feature" self.chunk_dims = [d.upper() for d in self.chunk_dims] + + if dim_order is not None: + self._dim_order = dim_order + else: + self._dim_order = "".join(d for d in DEFAULT_DIMENSION_ORDER if d in self._all_files.columns or d in self.chunk_dims) - self._dim_order = dim_order self._channel_names = channel_names if single_file_shape is None: @@ -262,9 +267,7 @@ def _read_delayed(self) -> xr.DataArray: dims, d_data.shape, self.current_scene_index, channel_names ) x_data = xr.DataArray(d_data, dims=dims, coords=coords) - if self._dim_order is not None: - x_data.transpose(*list(self._dim_order)) - + return x_data def _read_immediate(self) -> xr.DataArray: @@ -295,8 +298,6 @@ def _read_immediate(self) -> xr.DataArray: dims=dims, coords=coords, ) - if self._dim_order is not None: - x_data.transpose(*list(self._dim_order)) return x_data From 0e51b14a34eb9ddf6d044c9b396186f4bbf87431 Mon Sep 17 00:00:00 2001 From: John Russell Date: Thu, 2 Sep 2021 14:10:06 -0400 Subject: [PATCH 11/29] Implement tests for multidimensionsal image files. test 3d case (and commented broken test for 4d) 4d images working immediate reads and certain chunks test 3d and some 4d cases tests passing of 2,3,4 d images --- aicsimageio/readers/glob_reader.py | 73 ++++++++++-- aicsimageio/tests/readers/test_glob_reader.py | 112 ++++++++++++++++-- 2 files changed, 160 insertions(+), 25 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index a0e160d89..02f501f72 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -126,7 +126,6 @@ def __init__( self._all_files[dim] = 0 if dim in self._all_files.columns: sort_order.append(dim) - self._all_files = self._all_files.sort_values(sort_order).reset_index(drop=True) # run tests on a single file (?) @@ -230,33 +229,63 @@ def _read_delayed(self) -> xr.DataArray: ] group_sizes = OrderedDict([(d, scene_nunique[d]) for d in group_dims]) - chunk_sizes = self._get_sizes_for_scene(scene_nunique, group_dims) - - + chunk_sizes = self._get_chunk_sizes(scene_nunique, group_dims) + unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in set(chunk_sizes.keys())-set(group_sizes.keys())]) + reshape_sizes = tuple(unpack_sizes.values())+tuple(self._single_file_sizes.values()) + # reshape_axes = tuple(unpack_sizes.keys()) + tuple(self._single_file_sizes.keys()) + + axes_order = self._get_axes_order(chunk_sizes, unpack_sizes, group_sizes) + + print(f"{group_sizes=}") + print(f"{chunk_sizes=}") + # print(f"{unpack_sizes=}") + # print(f"{self._dim_order=}") + # print(f"{self._single_file_sizes=}") + print(f"{reshape_sizes=}") + print(f"{axes_order=}") # Assemble the dask array if len(group_dims)>0: #use groupby to assemble array out of chunks chunks = np.zeros(tuple(group_sizes.values()), dtype="object") - for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): + # print(val.filename.values) zarr_im = imread(val.filename.tolist(), aszarr=True) darr = da.from_zarr(zarr_im).rechunk(-1) + if i==0: print("orignal" ,darr.shape) + # simply reshapeing here is not specific enough, may need to swap axes first? + # trying to do crazy stiff staring here + # unpack the first dimension if it contains multiple axes + darr = darr.reshape(reshape_sizes) + # Then reorder dimensions so matching ones from the glob and the file are adjacent (glob then file) + darr = darr.transpose(axes_order) + # end madness + # Then reshape the array to chunk_sizes darr = darr.reshape(tuple(chunk_sizes.values())) + if i==0: print("final", f"{darr.shape=}") chunks[idx] = darr - + print(f"{chunks.shape=}") overlap_dims = group_sizes.keys() & chunk_sizes.keys() + print(f"{overlap_dims=}") + # PROBLEM HERE IN SETUP OF EXPANDED SHAPE + #chunks_axes = tuple(group_sizes.keys())+tuple(chunk_sizes.keys()) + expanded_shape = chunks.shape + tuple([1 for d in chunk_sizes if d not in group_sizes]) + #print(f"{chunks_axes=}") expanded_shape = tuple(s for d,s in group_sizes.items() if d not in overlap_dims) + #expanded_shape = (2,1,1,5,1,1)#(2,5,1,1,1,1) + #print(expanded_shape) for d in chunk_sizes: if d in overlap_dims: expanded_shape += (group_sizes[d],) else: expanded_shape += (1,) - + # print(f"{expanded_shape=}") chunks = chunks.reshape(expanded_shape) d_data = da.block(chunks.tolist()) - + print(d_data.shape) else: # assemble array in a single chunk zarr_im = imread(scene_files.filename.tolist(), aszarr=True) darr = da.from_zarr(zarr_im).rechunk(-1) + darr = darr.reshape(reshape_sizes) + darr = darr.transpose(axes_order) d_data = darr.reshape(tuple(chunk_sizes.values())) # Assign dims and coords to construct xarray @@ -267,7 +296,9 @@ def _read_delayed(self) -> xr.DataArray: dims, d_data.shape, self.current_scene_index, channel_names ) x_data = xr.DataArray(d_data, dims=dims, coords=coords) - + + x_data = x_data.transpose(*self._dim_order) + return x_data def _read_immediate(self) -> xr.DataArray: @@ -278,11 +309,18 @@ def _read_immediate(self) -> xr.DataArray: scene_files = scene_files.drop(DimensionNames.Samples, axis=1) scene_nunique = scene_files.nunique() - scene_sizes = self._get_sizes_for_scene(scene_nunique) + chunk_sizes = self._get_chunk_sizes(scene_nunique) + + unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in chunk_sizes.keys()]) + + reshape_sizes = tuple(unpack_sizes.values())+tuple(self._single_file_sizes.values()) + axes_order = self._get_axes_order(chunk_sizes, unpack_sizes) # Assemble array arr = imread(scene_files.filename.tolist()) - arr = arr.reshape(tuple(scene_sizes.values())) + arr = arr.reshape(reshape_sizes) + arr = arr.transpose(axes_order) + arr = arr.reshape(tuple(chunk_sizes.values())) # Assign dims and coords to construct xarray dims = scene_files.columns.drop("filename").values.tolist() @@ -301,9 +339,18 @@ def _read_immediate(self) -> xr.DataArray: return x_data - def _get_sizes_for_scene( + def _get_axes_order(self, chunk_sizes: OrderedDict, unpack_sizes: OrderedDict, group_sizes: OrderedDict = OrderedDict()) -> Tuple : + axes_order = () + for d in chunk_sizes: + if d in unpack_sizes: + axes_order += (list(unpack_sizes.keys()).index(d),) + if d in self._single_file_sizes: + axes_order += (len(unpack_sizes) + list(self._single_file_sizes.keys()).index(d),) + return axes_order + + def _get_chunk_sizes( self, scene_files_nunique: pd.Series, group_dims: List[str] = [] - ): + ) -> OrderedDict : sizes = OrderedDict() for i, x in scene_files_nunique.iteritems(): diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index 7c13d7f5e..a1dabb0cd 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -1,42 +1,130 @@ #! usr/env/bin/python import os +import re import numpy as np import xarray as xr +import pandas as pd import tifffile as tiff import aicsimageio from pathlib import Path -TIFF_3D = False -DATA_SHAPE = (2,3,4,5,6,7) # STCZYX +DATA_SHAPE = (3,4,5,6,7,8) # STCZYX +def check_values(reader, reference): + for i, s in enumerate(reader.scenes): + reader.set_scene(s) + assert np.all(reference.isel(S=i).data == reader.xarray_dask_data.data).compute() + assert np.all(reference.isel(S=i).data == reader.xarray_data.data) def make_fake_data_2d(path): data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) dims = list("STCZYX") - image_ids = [aicsimageio.metadata.utils.generate_ome_image_id(i) for i in range(DATA_SHAPE[0])] - x_data = xr.DataArray(data, dims=dims)#, coords=coords) + x_data = xr.DataArray(data, dims=dims) + os.mkdir(str(path/"2d_images")) + for s in range(DATA_SHAPE[0]): for t in range(DATA_SHAPE[1]): for c in range(DATA_SHAPE[2]): for z in range(DATA_SHAPE[3]): im = data[s,t,c,z] - tiff.imsave(str(path/f"S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16) + tiff.imsave(str(path/f"2d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16) return x_data def test_glob_reader_2d(tmp_path: Path): reference = make_fake_data_2d(tmp_path) - gr = aicsimageio.readers.GlobReader(str(tmp_path/"*.tif")) + gr = aicsimageio.readers.GlobReader(str(tmp_path/"2d_images/*.tif")) + + assert gr.xarray_dask_data.data.chunksize == (1,1)+DATA_SHAPE[-3:] + + check_values(gr, reference) + +def make_fake_data_3d(path): + + data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) + + dims = list("STCZYX") + + x_data = xr.DataArray(data, dims=dims) + + os.mkdir(str(path/"3d_images")) + + for s in range(DATA_SHAPE[0]): + for t in range(DATA_SHAPE[1]): + for c in range(DATA_SHAPE[2]): + for z in range(int(DATA_SHAPE[3]/2)): + im = data[s,t,c,2*z:2*(z+1)] + tiff.imsave(str(path/f"3d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16) + return x_data + +def test_glob_reader_3d(tmp_path: Path): + reference = make_fake_data_3d(tmp_path) + idxr = lambda x: pd.Series(re.findall(r"\d+", Path(x).name), index=list('STC')).astype(int) + + #do not stack z dimension + gr = aicsimageio.readers.GlobReader(str(tmp_path/"3d_images/*Z0.tif"), single_file_dims=list('ZYX')) + assert gr.xarray_dask_data.data.chunksize == (1,1,2,7,8) + check_values(gr, reference.isel(Z=slice(0,2))) + + # stack along z dimension but do not chunk + gr = aicsimageio.readers.GlobReader(str(tmp_path/"3d_images/*.tif"), single_file_dims=list('ZYX'), chunk_dims=list('TC')) + assert gr.xarray_dask_data.data.chunksize == (4,5,2,7,8) + check_values(gr, reference) + + #stack along z and chunk along z + gr = aicsimageio.readers.GlobReader(str(tmp_path/"3d_images/*.tif"), single_file_dims=list('ZYX')) + assert gr.xarray_dask_data.data.chunksize == (1,1,6,7,8) + check_values(gr, reference) + +def make_fake_data_4d(path): + + data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) + + dims = list("STCZYX") + + x_data = xr.DataArray(data, dims=dims) + + os.mkdir(str(path/"4d_images")) + + per_file_t = 2 + t_files = int(DATA_SHAPE[1]/per_file_t) + + per_file_z = 3 + z_files = int(DATA_SHAPE[3]/per_file_z) + + + for s in range(DATA_SHAPE[0]): + for t in range(t_files): + for c in range(DATA_SHAPE[2]): + for z in range(z_files): + im = data[s,per_file_t*t:per_file_t*(t+1),c,per_file_z*z:per_file_z*(z+1)] + tiff.imsave(str(path/f"4d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16, photometric='MINISBLACK') + return x_data + +def test_glob_reader_4d(tmp_path: Path): + reference = make_fake_data_4d(tmp_path) + + # stack none + gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*T0*Z0.tif"), single_file_dims=list('TZYX')) + assert gr.xarray_dask_data.data.chunksize == (2,1,3,7,8) + check_values(gr, reference.isel(T=slice(0,2), Z=slice(0,3))) - for i, s in enumerate(gr.scenes): - gr.set_scene(s) - assert np.all(reference.isel(S=i).data == gr.xarray_dask_data.data) - assert np.all(reference.isel(S=i).data == gr.xarray_data.data) + # stack z and t - chunk z + gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*.tif"), single_file_dims=list('TZYX')) + assert gr.xarray_dask_data.data.chunksize == (2,1,6,7,8) + check_values(gr, reference) + # stack z and t - chunk z and t + gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*.tif"), single_file_dims=list('TZYX'), chunk_dims=['T','Z']) + assert gr.xarray_dask_data.data.chunksize == (4,1,6,7,8) + check_values(gr, reference) + # stack z an t - chunk ztc + gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*.tif"), single_file_dims=list('TZYX'),\ + chunk_dims=list('TCZ')) + assert gr.xarray_dask_data.data.chunksize==(4,5,6,7,8) + check_values(gr, reference) -# im = data[s,t,c] -# tiff.imsave(f"fake_data/S{s}_T{t}_C{c}.tif", im, dtype=np.uint16) From a030ec05a326ad8b9f1b61cc74c1fe95ab942269 Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 15:01:51 -0400 Subject: [PATCH 12/29] fix reshaping of chunks to handle multidimensional single files --- aicsimageio/readers/glob_reader.py | 99 ++++++++++++++++++------------ 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 02f501f72..291004058 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -126,6 +126,7 @@ def __init__( self._all_files[dim] = 0 if dim in self._all_files.columns: sort_order.append(dim) + self._all_files = self._all_files.sort_values(sort_order).reset_index(drop=True) # run tests on a single file (?) @@ -133,7 +134,11 @@ def __init__( # Store params if isinstance(chunk_dims, str): - chunk_dims = list(chunk_dims) + self.chunk_dims = list(chunk_dims) + elif isinstance(chunk_dims, list) and isinstance(chunk_dims[0], str): + self.chunk_dims = chunk_dims + else: + raise ValueError("chunk_dims must be str or list of str") # Run basic checks on dims and channel names if isinstance(dim_order, list): @@ -159,7 +164,7 @@ def __init__( ) self._channel_names = channel_names - self.chunk_dims = chunk_dims + # self.chunk_dims = chunk_dims for dim in REQUIRED_CHUNK_DIMS: if dim not in self.chunk_dims: @@ -230,66 +235,45 @@ def _read_delayed(self) -> xr.DataArray: group_sizes = OrderedDict([(d, scene_nunique[d]) for d in group_dims]) chunk_sizes = self._get_chunk_sizes(scene_nunique, group_dims) - unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in set(chunk_sizes.keys())-set(group_sizes.keys())]) + unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in chunk_sizes.keys()-group_sizes.keys()]) reshape_sizes = tuple(unpack_sizes.values())+tuple(self._single_file_sizes.values()) - # reshape_axes = tuple(unpack_sizes.keys()) + tuple(self._single_file_sizes.keys()) - + axes_order = self._get_axes_order(chunk_sizes, unpack_sizes, group_sizes) - print(f"{group_sizes=}") - print(f"{chunk_sizes=}") - # print(f"{unpack_sizes=}") - # print(f"{self._dim_order=}") - # print(f"{self._single_file_sizes=}") - print(f"{reshape_sizes=}") - print(f"{axes_order=}") + + expanded_blocks_sizes, expanded_chunks_sizes = self._get_expanded_shapes(group_sizes, chunk_sizes) + # Assemble the dask array if len(group_dims)>0: #use groupby to assemble array out of chunks chunks = np.zeros(tuple(group_sizes.values()), dtype="object") for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): - # print(val.filename.values) zarr_im = imread(val.filename.tolist(), aszarr=True) darr = da.from_zarr(zarr_im).rechunk(-1) - if i==0: print("orignal" ,darr.shape) - # simply reshapeing here is not specific enough, may need to swap axes first? - # trying to do crazy stiff staring here + # unpack the first dimension if it contains multiple axes darr = darr.reshape(reshape_sizes) + # Then reorder dimensions so matching ones from the glob and the file are adjacent (glob then file) darr = darr.transpose(axes_order) - # end madness + # Then reshape the array to chunk_sizes - darr = darr.reshape(tuple(chunk_sizes.values())) - if i==0: print("final", f"{darr.shape=}") + darr = darr.reshape(tuple(expanded_chunks_sizes.values())) + chunks[idx] = darr - print(f"{chunks.shape=}") - overlap_dims = group_sizes.keys() & chunk_sizes.keys() - print(f"{overlap_dims=}") - # PROBLEM HERE IN SETUP OF EXPANDED SHAPE - #chunks_axes = tuple(group_sizes.keys())+tuple(chunk_sizes.keys()) - expanded_shape = chunks.shape + tuple([1 for d in chunk_sizes if d not in group_sizes]) - #print(f"{chunks_axes=}") - expanded_shape = tuple(s for d,s in group_sizes.items() if d not in overlap_dims) - #expanded_shape = (2,1,1,5,1,1)#(2,5,1,1,1,1) - #print(expanded_shape) - for d in chunk_sizes: - if d in overlap_dims: - expanded_shape += (group_sizes[d],) - else: - expanded_shape += (1,) - # print(f"{expanded_shape=}") - chunks = chunks.reshape(expanded_shape) + + chunks = chunks.reshape(tuple(expanded_blocks_sizes.values())) d_data = da.block(chunks.tolist()) - print(d_data.shape) + dims = list(expanded_blocks_sizes.keys()) + else: # assemble array in a single chunk zarr_im = imread(scene_files.filename.tolist(), aszarr=True) darr = da.from_zarr(zarr_im).rechunk(-1) darr = darr.reshape(reshape_sizes) darr = darr.transpose(axes_order) d_data = darr.reshape(tuple(chunk_sizes.values())) - + dims = list(expanded_chunks_sizes.keys()) + # Assign dims and coords to construct xarray - dims = [d for d in group_dims if d not in overlap_dims] + list(chunk_sizes.keys()) channel_names = self._get_channel_names_for_scene(dims) coords = self._get_coords( @@ -369,6 +353,43 @@ def _get_chunk_sizes( sizes[i] = x return sizes + + def _get_expanded_shapes(self, group_sizes: OrderedDict, chunk_sizes: OrderedDict) -> Tuple[OrderedDict, OrderedDict] : + expanded_blocks_sizes = OrderedDict() + expanded_chunks_sizes = OrderedDict() + + for i,(d,s) in enumerate(group_sizes.items()): + if d in chunk_sizes : + if d not in expanded_blocks_sizes: + d_idx_in_chunks = list(chunk_sizes.keys()).index(d) + for j in range(d_idx_in_chunks): + c_key = list(chunk_sizes.keys())[j] + if c_key not in expanded_blocks_sizes: + expanded_blocks_sizes[c_key] = 1 + + expanded_blocks_sizes[d] = s + + if d not in chunk_sizes: + if len(expanded_blocks_sizes)<=i: + expanded_blocks_sizes[d] = s + else: + for d2 in expanded_blocks_sizes: + expanded_chunks_sizes[d2] = chunk_sizes[d2] + expanded_chunks_sizes[d]=1 + expanded_blocks_sizes[d]=group_sizes[d] + for d2, s2 in chunk_sizes.items(): + if d2 not in expanded_chunks_sizes: + expanded_chunks_sizes[d2]=s2 + + for d,s in chunk_sizes.items(): + if d not in expanded_blocks_sizes: + expanded_blocks_sizes[d] = 1 + + + if len(expanded_chunks_sizes)==0: + expanded_chunks_sizes = chunk_sizes + + return expanded_blocks_sizes, expanded_chunks_sizes def _get_channel_names_for_scene(self, dims: List[str]) -> Optional[List[str]]: # Fast return in None case From e0bc6f4008738caa4c77df8784a046330c72d8ab Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 15:19:59 -0400 Subject: [PATCH 13/29] process metadata from single file for each scene --- aicsimageio/readers/glob_reader.py | 42 +++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 291004058..71fe9b5b2 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -29,6 +29,8 @@ from .reader import Reader +TIFF_IMAGE_DESCRIPTION_TAG_INDEX = 270 + class GlobReader(Reader): """ @@ -229,6 +231,9 @@ def _read_delayed(self) -> xr.DataArray: ] scene_files = scene_files.drop(DimensionNames.Samples, axis=1) scene_nunique = scene_files.nunique() + + tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) + group_dims = [ x for x in scene_files.columns if x not in self.chunk_dims + ["filename"] ] @@ -279,7 +284,19 @@ def _read_delayed(self) -> xr.DataArray: coords = self._get_coords( dims, d_data.shape, self.current_scene_index, channel_names ) - x_data = xr.DataArray(d_data, dims=dims, coords=coords) + + # Try accepted processed metadata + try: + attrs = { + constants.METADATA_UNPROCESSED: tiff_tags, + constants.METADATA_PROCESSED: tiff_tags[ + TIFF_IMAGE_DESCRIPTION_TAG_INDEX + ], + } + except KeyError: + attrs = {constants.METADATA_UNPROCESSED: tiff_tags} + + x_data = xr.DataArray(d_data, dims=dims, coords=coords, attrs=attrs) x_data = x_data.transpose(*self._dim_order) @@ -293,6 +310,8 @@ def _read_immediate(self) -> xr.DataArray: scene_files = scene_files.drop(DimensionNames.Samples, axis=1) scene_nunique = scene_files.nunique() + tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) + chunk_sizes = self._get_chunk_sizes(scene_nunique) unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in chunk_sizes.keys()]) @@ -315,6 +334,16 @@ def _read_immediate(self) -> xr.DataArray: coords = self._get_coords(dims, arr.shape, self.current_scene_index, channel_names) + # Try accepted processed metadata + try: + attrs = { + constants.METADATA_UNPROCESSED: tiff_tags, + constants.METADATA_PROCESSED: tiff_tags[ + TIFF_IMAGE_DESCRIPTION_TAG_INDEX + ], + } + except KeyError: + attrs = {constants.METADATA_UNPROCESSED: tiff_tags} x_data = xr.DataArray( arr, dims=dims, @@ -450,3 +479,14 @@ def _get_coords( coords[DimensionNames.Channel] = channel_names return coords + + + def _get_tiff_tags(self, tiff: TiffFile) -> TiffTags: + unprocessed_tags = tiff.series[0].pages[0].tags + + # Create dict of tag and value + tags: Dict[int, str] = {} + for code, tag in unprocessed_tags.items(): + tags[code] = tag.value + + return tags From 3ca021827b563e9f5394092e95d9041bcad21bb7 Mon Sep 17 00:00:00 2001 From: John Russell <35578729+jrussell25@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:21:26 -0400 Subject: [PATCH 14/29] Update aicsimageio/readers/glob_reader.py fix typo in comments Co-authored-by: Madison Swain-Bowden --- aicsimageio/readers/glob_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 71fe9b5b2..eea29eccf 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -95,7 +95,7 @@ def __init__( **kwargs: Any, ): - # Assmble glob list if given a string + # Assemble glob list if given a string if isinstance(glob_in, str): file_series = pd.Series(glob.glob(glob_in)) elif isinstance(glob_in, list): From f75cf06b09aa9a3eb367bd11cb280dfbfe565e8d Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 15:42:31 -0400 Subject: [PATCH 15/29] Implement changes from discussion. - Add glob_scene_character argument that defaults to "S" - Add level and chunkmode arguments to imread calls - Add some comments around defualt indexer function --- aicsimageio/readers/glob_reader.py | 32 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index eea29eccf..4503a41d0 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -45,6 +45,9 @@ class GlobReader(Reader): indexer: Union[Callable, pandas.DataFrame] If callable, should consume each filename and return a pd.Series with index corresponding to the dimensions and values corresponding to the index + scene_glob_character: str + Character to represent different scenes. + Default: "S" chunk_dims: Union[str, List[str]] Which dimensions to create chunks for. Default: DEFAULT_CHUNK_DIMS @@ -84,6 +87,7 @@ def __init__( self, glob_in: Union[str, List[str]], indexer: Union[pd.DataFrame, Callable] = None, + scene_glob_character: str = "S", chunk_dims: Union[str, List[str]] = DEFAULT_CHUNK_DIMS, dim_order: Optional[Union[List[str], str]] = None, channel_names: Optional[Union[List[str], List[List[str]]]] = None, @@ -99,18 +103,24 @@ def __init__( if isinstance(glob_in, str): file_series = pd.Series(glob.glob(glob_in)) elif isinstance(glob_in, list): - file_series = pd.Series(glob_in) # pd.DataFrame({"filename": glob_in}) + file_series = pd.Series(glob_in) if len(file_series) == 0: raise ValueError("No files found matching glob pattern") + self.scene_glob_character = scene_glob_character + if indexer is None: series_idx = [ - DimensionNames.Samples, + self.scene_glob_character, DimensionNames.Time, DimensionNames.Channel, DimensionNames.SpatialZ, ] + + # By default we will attempt to parse 4 numbers out of the filename + # and assign them in order to be the corresponding S, T, C, and Z indices. + # So indexer("S0_T1_C2_Z3.tif") returns pd.Series([0,1,2,3], index=['S','T','C', 'Z']) indexer = lambda x: pd.Series( re.findall(r"\d+", Path(x).name), index=series_idx ).astype(int) @@ -220,22 +230,22 @@ def scenes(self) -> Tuple[str, ...]: if self._scenes is None: self._scenes = tuple( metadata_utils.generate_ome_image_id(s) - for s in range(self._all_files[DimensionNames.Samples].nunique()) + for s in range(self._all_files[scene_glob_character].nunique()) ) return self._scenes def _read_delayed(self) -> xr.DataArray: scene_files = self._all_files.loc[ - self._all_files[DimensionNames.Samples] == self.current_scene_index + self._all_files[self.scene_glob_character] == self.current_scene_index ] - scene_files = scene_files.drop(DimensionNames.Samples, axis=1) + scene_files = scene_files.drop(self.scene_glob_character, axis=1) scene_nunique = scene_files.nunique() tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) group_dims = [ - x for x in scene_files.columns if x not in self.chunk_dims + ["filename"] + x for x in scene_files.columns if x not in ["filename", *self.chunk_dims] ] group_sizes = OrderedDict([(d, scene_nunique[d]) for d in group_dims]) @@ -252,7 +262,7 @@ def _read_delayed(self) -> xr.DataArray: if len(group_dims)>0: #use groupby to assemble array out of chunks chunks = np.zeros(tuple(group_sizes.values()), dtype="object") for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): - zarr_im = imread(val.filename.tolist(), aszarr=True) + zarr_im = imread(val.filename.tolist(), aszarr=True, level=0, chunkmode="page") darr = da.from_zarr(zarr_im).rechunk(-1) # unpack the first dimension if it contains multiple axes @@ -271,7 +281,7 @@ def _read_delayed(self) -> xr.DataArray: dims = list(expanded_blocks_sizes.keys()) else: # assemble array in a single chunk - zarr_im = imread(scene_files.filename.tolist(), aszarr=True) + zarr_im = imread(scene_files.filename.tolist(), aszarr=True, level=0, chunkmode="page") darr = da.from_zarr(zarr_im).rechunk(-1) darr = darr.reshape(reshape_sizes) darr = darr.transpose(axes_order) @@ -305,9 +315,9 @@ def _read_delayed(self) -> xr.DataArray: def _read_immediate(self) -> xr.DataArray: # Set up scene specific information scene_files = self._all_files.loc[ - self._all_files[DimensionNames.Samples] == self.current_scene_index + self._all_files[self.scene_glob_character] == self.current_scene_index ] - scene_files = scene_files.drop(DimensionNames.Samples, axis=1) + scene_files = scene_files.drop(self.scene_glob_character, axis=1) scene_nunique = scene_files.nunique() tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) @@ -320,7 +330,7 @@ def _read_immediate(self) -> xr.DataArray: axes_order = self._get_axes_order(chunk_sizes, unpack_sizes) # Assemble array - arr = imread(scene_files.filename.tolist()) + arr = imread(scene_files.filename.tolist(), level=0, chunkmode="page") arr = arr.reshape(reshape_sizes) arr = arr.transpose(axes_order) arr = arr.reshape(tuple(chunk_sizes.values())) From 28aba420b24ed78e1159f14a659bc4378eec61d7 Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 16:35:34 -0400 Subject: [PATCH 16/29] add self --- aicsimageio/readers/glob_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 4503a41d0..e274c7cae 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -230,7 +230,7 @@ def scenes(self) -> Tuple[str, ...]: if self._scenes is None: self._scenes = tuple( metadata_utils.generate_ome_image_id(s) - for s in range(self._all_files[scene_glob_character].nunique()) + for s in range(self._all_files[self.scene_glob_character].nunique()) ) return self._scenes From 115856297199bed23dad1e1fbb9c6d1cb5426dbc Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 19:19:22 -0400 Subject: [PATCH 17/29] get rid of chunkmode kwarg in imread --- aicsimageio/readers/glob_reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index e274c7cae..7f8486d95 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -262,7 +262,7 @@ def _read_delayed(self) -> xr.DataArray: if len(group_dims)>0: #use groupby to assemble array out of chunks chunks = np.zeros(tuple(group_sizes.values()), dtype="object") for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): - zarr_im = imread(val.filename.tolist(), aszarr=True, level=0, chunkmode="page") + zarr_im = imread(val.filename.tolist(), aszarr=True, level=0) darr = da.from_zarr(zarr_im).rechunk(-1) # unpack the first dimension if it contains multiple axes @@ -281,7 +281,7 @@ def _read_delayed(self) -> xr.DataArray: dims = list(expanded_blocks_sizes.keys()) else: # assemble array in a single chunk - zarr_im = imread(scene_files.filename.tolist(), aszarr=True, level=0, chunkmode="page") + zarr_im = imread(scene_files.filename.tolist(), aszarr=True, level=0) darr = da.from_zarr(zarr_im).rechunk(-1) darr = darr.reshape(reshape_sizes) darr = darr.transpose(axes_order) @@ -330,7 +330,7 @@ def _read_immediate(self) -> xr.DataArray: axes_order = self._get_axes_order(chunk_sizes, unpack_sizes) # Assemble array - arr = imread(scene_files.filename.tolist(), level=0, chunkmode="page") + arr = imread(scene_files.filename.tolist(), level=0) arr = arr.reshape(reshape_sizes) arr = arr.transpose(axes_order) arr = arr.reshape(tuple(chunk_sizes.values())) From ca37b7e45bced17652f1bfa7cb94a79c089e32da Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 20:23:54 -0400 Subject: [PATCH 18/29] more cleanup including formatting. more comments --- aicsimageio/readers/glob_reader.py | 135 +++++++++++++++++++---------- 1 file changed, 87 insertions(+), 48 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index 7f8486d95..ef3979cb6 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -31,6 +31,7 @@ TIFF_IMAGE_DESCRIPTION_TAG_INDEX = 270 + class GlobReader(Reader): """ @@ -43,9 +44,10 @@ class GlobReader(Reader): Glob string that identifies all files to be loaded or a list of paths to the files as returned by glob. indexer: Union[Callable, pandas.DataFrame] - If callable, should consume each filename and return a pd.Series with index - corresponding to the dimensions and values corresponding to the index - scene_glob_character: str + If callable, should consume each filename and return a pd.Series with series index + corresponding to the dimensions and values corresponding to the array index of that image file within the larger array. + Default: None (Look for 4 numbers in the file name and use them as S, T, C, and Z indices. + scene_glob_character: str Character to represent different scenes. Default: "S" chunk_dims: Union[str, List[str]] @@ -120,7 +122,7 @@ def __init__( # By default we will attempt to parse 4 numbers out of the filename # and assign them in order to be the corresponding S, T, C, and Z indices. - # So indexer("S0_T1_C2_Z3.tif") returns pd.Series([0,1,2,3], index=['S','T','C', 'Z']) + # So indexer("path/to/data/S0_T1_C2_Z3.tif") returns pd.Series([0,1,2,3], index=['S','T','C', 'Z']) indexer = lambda x: pd.Series( re.findall(r"\d+", Path(x).name), index=series_idx ).astype(int) @@ -138,7 +140,7 @@ def __init__( self._all_files[dim] = 0 if dim in self._all_files.columns: sort_order.append(dim) - + self._all_files = self._all_files.sort_values(sort_order).reset_index(drop=True) # run tests on a single file (?) @@ -176,19 +178,21 @@ def __init__( ) self._channel_names = channel_names - # self.chunk_dims = chunk_dims - for dim in REQUIRED_CHUNK_DIMS: if dim not in self.chunk_dims: self.chunk_dims.append(dim) # Safety measure / "feature" self.chunk_dims = [d.upper() for d in self.chunk_dims] - + if dim_order is not None: self._dim_order = dim_order else: - self._dim_order = "".join(d for d in DEFAULT_DIMENSION_ORDER if d in self._all_files.columns or d in self.chunk_dims) + self._dim_order = "".join( + d + for d in DEFAULT_DIMENSION_ORDER + if d in self._all_files.columns or d in self.chunk_dims + ) self._channel_names = channel_names @@ -241,25 +245,47 @@ def _read_delayed(self) -> xr.DataArray: ] scene_files = scene_files.drop(self.scene_glob_character, axis=1) scene_nunique = scene_files.nunique() - + tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) group_dims = [ x for x in scene_files.columns if x not in ["filename", *self.chunk_dims] ] + # xxx_sizes are modeled after xr.DataArray.sizes + # These are OrderedDicts that map a dimension name to a shape. + # Use these to align and reshape the arrays that come from imread + # dims and sizes are not always necessary but they keep things much + # clearer internally. + + # sizes of dimensions we grouping by i.e. not chunks group_sizes = OrderedDict([(d, scene_nunique[d]) for d in group_dims]) + + # sizes of each chunk chunk_sizes = self._get_chunk_sizes(scene_nunique, group_dims) - unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in chunk_sizes.keys()-group_sizes.keys()]) - reshape_sizes = tuple(unpack_sizes.values())+tuple(self._single_file_sizes.values()) - + + # sizes that will be used to reshape the array representing the full glob into separate dimensions + unpack_sizes = OrderedDict( + [ + (d, s) + for d, s in scene_nunique.iteritems() + if d in chunk_sizes.keys() - group_sizes.keys() + ] + ) + reshape_sizes = tuple(unpack_sizes.values()) + tuple( + self._single_file_sizes.values() + ) + + # after unpacking the result of imread we sometimes need to rearrange dims in case they are in the glob and single files axes_order = self._get_axes_order(chunk_sizes, unpack_sizes, group_sizes) - - expanded_blocks_sizes, expanded_chunks_sizes = self._get_expanded_shapes(group_sizes, chunk_sizes) + # expand the sizes with singleton dimensions to facilitate dask.array.block at the end + expanded_blocks_sizes, expanded_chunk_sizes = self._get_expanded_shapes( + group_sizes, chunk_sizes + ) # Assemble the dask array - if len(group_dims)>0: #use groupby to assemble array out of chunks + if len(group_dims) > 0: # use groupby to assemble array out of chunks chunks = np.zeros(tuple(group_sizes.values()), dtype="object") for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): zarr_im = imread(val.filename.tolist(), aszarr=True, level=0) @@ -272,7 +298,7 @@ def _read_delayed(self) -> xr.DataArray: darr = darr.transpose(axes_order) # Then reshape the array to chunk_sizes - darr = darr.reshape(tuple(expanded_chunks_sizes.values())) + darr = darr.reshape(tuple(expanded_chunk_sizes.values())) chunks[idx] = darr @@ -280,14 +306,14 @@ def _read_delayed(self) -> xr.DataArray: d_data = da.block(chunks.tolist()) dims = list(expanded_blocks_sizes.keys()) - else: # assemble array in a single chunk + else: # assemble array in a single chunk zarr_im = imread(scene_files.filename.tolist(), aszarr=True, level=0) darr = da.from_zarr(zarr_im).rechunk(-1) darr = darr.reshape(reshape_sizes) darr = darr.transpose(axes_order) d_data = darr.reshape(tuple(chunk_sizes.values())) dims = list(expanded_chunks_sizes.keys()) - + # Assign dims and coords to construct xarray channel_names = self._get_channel_names_for_scene(dims) @@ -307,7 +333,7 @@ def _read_delayed(self) -> xr.DataArray: attrs = {constants.METADATA_UNPROCESSED: tiff_tags} x_data = xr.DataArray(d_data, dims=dims, coords=coords, attrs=attrs) - + x_data = x_data.transpose(*self._dim_order) return x_data @@ -324,17 +350,21 @@ def _read_immediate(self) -> xr.DataArray: chunk_sizes = self._get_chunk_sizes(scene_nunique) - unpack_sizes = OrderedDict([(d,s) for d,s in scene_nunique.iteritems() if d in chunk_sizes.keys()]) + unpack_sizes = OrderedDict( + [(d, s) for d, s in scene_nunique.iteritems() if d in chunk_sizes.keys()] + ) + + reshape_sizes = tuple(unpack_sizes.values()) + tuple( + self._single_file_sizes.values() + ) - reshape_sizes = tuple(unpack_sizes.values())+tuple(self._single_file_sizes.values()) - axes_order = self._get_axes_order(chunk_sizes, unpack_sizes) # Assemble array arr = imread(scene_files.filename.tolist(), level=0) arr = arr.reshape(reshape_sizes) arr = arr.transpose(axes_order) arr = arr.reshape(tuple(chunk_sizes.values())) - + # Assign dims and coords to construct xarray dims = scene_files.columns.drop("filename").values.tolist() file_dims = [x for x in self._single_file_dims if x not in dims] @@ -342,7 +372,9 @@ def _read_immediate(self) -> xr.DataArray: channel_names = self._get_channel_names_for_scene(dims) - coords = self._get_coords(dims, arr.shape, self.current_scene_index, channel_names) + coords = self._get_coords( + dims, arr.shape, self.current_scene_index, channel_names + ) # Try accepted processed metadata try: @@ -362,28 +394,35 @@ def _read_immediate(self) -> xr.DataArray: return x_data - def _get_axes_order(self, chunk_sizes: OrderedDict, unpack_sizes: OrderedDict, group_sizes: OrderedDict = OrderedDict()) -> Tuple : + def _get_axes_order( + self, + chunk_sizes: OrderedDict, + unpack_sizes: OrderedDict, + group_sizes: OrderedDict = OrderedDict(), + ) -> Tuple: axes_order = () for d in chunk_sizes: if d in unpack_sizes: axes_order += (list(unpack_sizes.keys()).index(d),) if d in self._single_file_sizes: - axes_order += (len(unpack_sizes) + list(self._single_file_sizes.keys()).index(d),) + axes_order += ( + len(unpack_sizes) + list(self._single_file_sizes.keys()).index(d), + ) return axes_order def _get_chunk_sizes( - self, scene_files_nunique: pd.Series, group_dims: List[str] = [] - ) -> OrderedDict : + self, scene_files_nunique: pd.Series, group_dims: List[str] = [] + ) -> OrderedDict: - sizes = OrderedDict() + sizes = OrderedDict() for i, x in scene_files_nunique.iteritems(): - if i not in group_dims + ["filename"]: + if i not in ["filename", *group_dims]: if i not in self._single_file_dims: sizes[i] = x else: sizes[i] = self._single_file_sizes[i] * x - - for d,s in self._single_file_sizes.items(): + + for d, s in self._single_file_sizes.items(): if d not in self.chunk_dims and d not in sizes: sizes[d] = s @@ -392,40 +431,41 @@ def _get_chunk_sizes( sizes[i] = x return sizes - - def _get_expanded_shapes(self, group_sizes: OrderedDict, chunk_sizes: OrderedDict) -> Tuple[OrderedDict, OrderedDict] : + + def _get_expanded_shapes( + self, group_sizes: OrderedDict, chunk_sizes: OrderedDict + ) -> Tuple[OrderedDict, OrderedDict]: expanded_blocks_sizes = OrderedDict() expanded_chunks_sizes = OrderedDict() - - for i,(d,s) in enumerate(group_sizes.items()): - if d in chunk_sizes : + + for i, (d, s) in enumerate(group_sizes.items()): + if d in chunk_sizes: if d not in expanded_blocks_sizes: d_idx_in_chunks = list(chunk_sizes.keys()).index(d) for j in range(d_idx_in_chunks): c_key = list(chunk_sizes.keys())[j] if c_key not in expanded_blocks_sizes: expanded_blocks_sizes[c_key] = 1 - + expanded_blocks_sizes[d] = s if d not in chunk_sizes: - if len(expanded_blocks_sizes)<=i: + if len(expanded_blocks_sizes) <= i: expanded_blocks_sizes[d] = s else: for d2 in expanded_blocks_sizes: expanded_chunks_sizes[d2] = chunk_sizes[d2] - expanded_chunks_sizes[d]=1 - expanded_blocks_sizes[d]=group_sizes[d] + expanded_chunks_sizes[d] = 1 + expanded_blocks_sizes[d] = group_sizes[d] for d2, s2 in chunk_sizes.items(): if d2 not in expanded_chunks_sizes: - expanded_chunks_sizes[d2]=s2 - - for d,s in chunk_sizes.items(): + expanded_chunks_sizes[d2] = s2 + + for d, s in chunk_sizes.items(): if d not in expanded_blocks_sizes: expanded_blocks_sizes[d] = 1 - - if len(expanded_chunks_sizes)==0: + if len(expanded_chunks_sizes) == 0: expanded_chunks_sizes = chunk_sizes return expanded_blocks_sizes, expanded_chunks_sizes @@ -490,7 +530,6 @@ def _get_coords( return coords - def _get_tiff_tags(self, tiff: TiffFile) -> TiffTags: unprocessed_tags = tiff.series[0].pages[0].tags From 0f53cd1c848c4eba1e147822521ad883e0e2da1a Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 20:24:15 -0400 Subject: [PATCH 19/29] formatting --- aicsimageio/tests/readers/test_glob_reader.py | 154 ++++++++++++------ 1 file changed, 100 insertions(+), 54 deletions(-) diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index a1dabb0cd..cad1cf9e8 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -8,123 +8,169 @@ import aicsimageio from pathlib import Path -DATA_SHAPE = (3,4,5,6,7,8) # STCZYX +DATA_SHAPE = (3, 4, 5, 6, 7, 8) # STCZYX + def check_values(reader, reference): for i, s in enumerate(reader.scenes): reader.set_scene(s) - assert np.all(reference.isel(S=i).data == reader.xarray_dask_data.data).compute() - assert np.all(reference.isel(S=i).data == reader.xarray_data.data) + assert np.all( + reference.isel(S=i).data == reader.xarray_dask_data.data + ).compute() + assert np.all(reference.isel(S=i).data == reader.xarray_data.data) + def make_fake_data_2d(path): - data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) + data = np.arange(np.prod(DATA_SHAPE), dtype="uint16").reshape(DATA_SHAPE) dims = list("STCZYX") - + x_data = xr.DataArray(data, dims=dims) - - os.mkdir(str(path/"2d_images")) + + os.mkdir(str(path / "2d_images")) for s in range(DATA_SHAPE[0]): for t in range(DATA_SHAPE[1]): for c in range(DATA_SHAPE[2]): - for z in range(DATA_SHAPE[3]): - im = data[s,t,c,z] - tiff.imsave(str(path/f"2d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16) + for z in range(DATA_SHAPE[3]): + im = data[s, t, c, z] + tiff.imsave( + str(path / f"2d_images/S{s}_T{t}_C{c}_Z{z}.tif"), + im, + dtype=np.uint16, + ) return x_data + def test_glob_reader_2d(tmp_path: Path): reference = make_fake_data_2d(tmp_path) - gr = aicsimageio.readers.GlobReader(str(tmp_path/"2d_images/*.tif")) - - assert gr.xarray_dask_data.data.chunksize == (1,1)+DATA_SHAPE[-3:] + gr = aicsimageio.readers.GlobReader(str(tmp_path / "2d_images/*.tif")) + + assert gr.xarray_dask_data.data.chunksize == (1, 1) + DATA_SHAPE[-3:] check_values(gr, reference) + def make_fake_data_3d(path): - data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) + data = np.arange(np.prod(DATA_SHAPE), dtype="uint16").reshape(DATA_SHAPE) dims = list("STCZYX") - + x_data = xr.DataArray(data, dims=dims) - - os.mkdir(str(path/"3d_images")) + + os.mkdir(str(path / "3d_images")) for s in range(DATA_SHAPE[0]): for t in range(DATA_SHAPE[1]): for c in range(DATA_SHAPE[2]): - for z in range(int(DATA_SHAPE[3]/2)): - im = data[s,t,c,2*z:2*(z+1)] - tiff.imsave(str(path/f"3d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16) + for z in range(int(DATA_SHAPE[3] / 2)): + im = data[s, t, c, 2 * z : 2 * (z + 1)] + tiff.imsave( + str(path / f"3d_images/S{s}_T{t}_C{c}_Z{z}.tif"), + im, + dtype=np.uint16, + ) return x_data + def test_glob_reader_3d(tmp_path: Path): reference = make_fake_data_3d(tmp_path) - idxr = lambda x: pd.Series(re.findall(r"\d+", Path(x).name), index=list('STC')).astype(int) - - #do not stack z dimension - gr = aicsimageio.readers.GlobReader(str(tmp_path/"3d_images/*Z0.tif"), single_file_dims=list('ZYX')) - assert gr.xarray_dask_data.data.chunksize == (1,1,2,7,8) - check_values(gr, reference.isel(Z=slice(0,2))) + idxr = lambda x: pd.Series( + re.findall(r"\d+", Path(x).name), index=list("STC") + ).astype(int) + + # do not stack z dimension + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "3d_images/*Z0.tif"), single_file_dims=list("ZYX") + ) + assert gr.xarray_dask_data.data.chunksize == (1, 1, 2, 7, 8) + check_values(gr, reference.isel(Z=slice(0, 2))) # stack along z dimension but do not chunk - gr = aicsimageio.readers.GlobReader(str(tmp_path/"3d_images/*.tif"), single_file_dims=list('ZYX'), chunk_dims=list('TC')) - assert gr.xarray_dask_data.data.chunksize == (4,5,2,7,8) + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "3d_images/*.tif"), + single_file_dims=list("ZYX"), + chunk_dims=list("TC"), + ) + assert gr.xarray_dask_data.data.chunksize == (4, 5, 2, 7, 8) check_values(gr, reference) - - #stack along z and chunk along z - gr = aicsimageio.readers.GlobReader(str(tmp_path/"3d_images/*.tif"), single_file_dims=list('ZYX')) - assert gr.xarray_dask_data.data.chunksize == (1,1,6,7,8) + + # stack along z and chunk along z + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "3d_images/*.tif"), single_file_dims=list("ZYX") + ) + assert gr.xarray_dask_data.data.chunksize == (1, 1, 6, 7, 8) check_values(gr, reference) + def make_fake_data_4d(path): - data = np.arange(np.prod(DATA_SHAPE), dtype='uint16').reshape(DATA_SHAPE) + data = np.arange(np.prod(DATA_SHAPE), dtype="uint16").reshape(DATA_SHAPE) dims = list("STCZYX") - + x_data = xr.DataArray(data, dims=dims) - - os.mkdir(str(path/"4d_images")) + + os.mkdir(str(path / "4d_images")) per_file_t = 2 - t_files = int(DATA_SHAPE[1]/per_file_t) + t_files = int(DATA_SHAPE[1] / per_file_t) per_file_z = 3 - z_files = int(DATA_SHAPE[3]/per_file_z) - + z_files = int(DATA_SHAPE[3] / per_file_z) for s in range(DATA_SHAPE[0]): for t in range(t_files): for c in range(DATA_SHAPE[2]): for z in range(z_files): - im = data[s,per_file_t*t:per_file_t*(t+1),c,per_file_z*z:per_file_z*(z+1)] - tiff.imsave(str(path/f"4d_images/S{s}_T{t}_C{c}_Z{z}.tif"), im, dtype=np.uint16, photometric='MINISBLACK') + im = data[ + s, + per_file_t * t : per_file_t * (t + 1), + c, + per_file_z * z : per_file_z * (z + 1), + ] + tiff.imsave( + str(path / f"4d_images/S{s}_T{t}_C{c}_Z{z}.tif"), + im, + dtype=np.uint16, + photometric="MINISBLACK", + ) return x_data + def test_glob_reader_4d(tmp_path: Path): reference = make_fake_data_4d(tmp_path) - + # stack none - gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*T0*Z0.tif"), single_file_dims=list('TZYX')) - assert gr.xarray_dask_data.data.chunksize == (2,1,3,7,8) - check_values(gr, reference.isel(T=slice(0,2), Z=slice(0,3))) + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "4d_images/*T0*Z0.tif"), single_file_dims=list("TZYX") + ) + assert gr.xarray_dask_data.data.chunksize == (2, 1, 3, 7, 8) + check_values(gr, reference.isel(T=slice(0, 2), Z=slice(0, 3))) # stack z and t - chunk z - gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*.tif"), single_file_dims=list('TZYX')) - assert gr.xarray_dask_data.data.chunksize == (2,1,6,7,8) + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "4d_images/*.tif"), single_file_dims=list("TZYX") + ) + assert gr.xarray_dask_data.data.chunksize == (2, 1, 6, 7, 8) check_values(gr, reference) # stack z and t - chunk z and t - gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*.tif"), single_file_dims=list('TZYX'), chunk_dims=['T','Z']) - assert gr.xarray_dask_data.data.chunksize == (4,1,6,7,8) + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "4d_images/*.tif"), + single_file_dims=list("TZYX"), + chunk_dims=["T", "Z"], + ) + assert gr.xarray_dask_data.data.chunksize == (4, 1, 6, 7, 8) check_values(gr, reference) - # stack z an t - chunk ztc - gr = aicsimageio.readers.GlobReader(str(tmp_path/"4d_images/*.tif"), single_file_dims=list('TZYX'),\ - chunk_dims=list('TCZ')) - assert gr.xarray_dask_data.data.chunksize==(4,5,6,7,8) + # stack z an t - chunk ztc + gr = aicsimageio.readers.GlobReader( + str(tmp_path / "4d_images/*.tif"), + single_file_dims=list("TZYX"), + chunk_dims=list("TCZ"), + ) + assert gr.xarray_dask_data.data.chunksize == (4, 5, 6, 7, 8) check_values(gr, reference) - From b468101bd76a093d10358d92c9692ef04da29f96 Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 8 Sep 2021 20:27:49 -0400 Subject: [PATCH 20/29] finish renaming some variables --- aicsimageio/readers/glob_reader.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/glob_reader.py index ef3979cb6..b2a3f1619 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/glob_reader.py @@ -286,7 +286,7 @@ def _read_delayed(self) -> xr.DataArray: # Assemble the dask array if len(group_dims) > 0: # use groupby to assemble array out of chunks - chunks = np.zeros(tuple(group_sizes.values()), dtype="object") + blocks = np.zeros(tuple(group_sizes.values()), dtype="object") for i, (idx, val) in enumerate(scene_files.groupby(group_dims)): zarr_im = imread(val.filename.tolist(), aszarr=True, level=0) darr = da.from_zarr(zarr_im).rechunk(-1) @@ -300,10 +300,10 @@ def _read_delayed(self) -> xr.DataArray: # Then reshape the array to chunk_sizes darr = darr.reshape(tuple(expanded_chunk_sizes.values())) - chunks[idx] = darr + blocks[idx] = darr - chunks = chunks.reshape(tuple(expanded_blocks_sizes.values())) - d_data = da.block(chunks.tolist()) + blocks = blocks.reshape(tuple(expanded_blocks_sizes.values())) + d_data = da.block(blocks.tolist()) dims = list(expanded_blocks_sizes.keys()) else: # assemble array in a single chunk @@ -312,7 +312,7 @@ def _read_delayed(self) -> xr.DataArray: darr = darr.reshape(reshape_sizes) darr = darr.transpose(axes_order) d_data = darr.reshape(tuple(chunk_sizes.values())) - dims = list(expanded_chunks_sizes.keys()) + dims = list(expanded_chunk_sizes.keys()) # Assign dims and coords to construct xarray channel_names = self._get_channel_names_for_scene(dims) @@ -436,7 +436,7 @@ def _get_expanded_shapes( self, group_sizes: OrderedDict, chunk_sizes: OrderedDict ) -> Tuple[OrderedDict, OrderedDict]: expanded_blocks_sizes = OrderedDict() - expanded_chunks_sizes = OrderedDict() + expanded_chunk_sizes = OrderedDict() for i, (d, s) in enumerate(group_sizes.items()): if d in chunk_sizes: @@ -454,21 +454,21 @@ def _get_expanded_shapes( expanded_blocks_sizes[d] = s else: for d2 in expanded_blocks_sizes: - expanded_chunks_sizes[d2] = chunk_sizes[d2] - expanded_chunks_sizes[d] = 1 + expanded_chunk_sizes[d2] = chunk_sizes[d2] + expanded_chunk_sizes[d] = 1 expanded_blocks_sizes[d] = group_sizes[d] for d2, s2 in chunk_sizes.items(): - if d2 not in expanded_chunks_sizes: - expanded_chunks_sizes[d2] = s2 + if d2 not in expanded_chunk_sizes: + expanded_chunk_sizes[d2] = s2 for d, s in chunk_sizes.items(): if d not in expanded_blocks_sizes: expanded_blocks_sizes[d] = 1 - if len(expanded_chunks_sizes) == 0: - expanded_chunks_sizes = chunk_sizes + if len(expanded_chunk_sizes) == 0: + expanded_chunk_sizes = chunk_sizes - return expanded_blocks_sizes, expanded_chunks_sizes + return expanded_blocks_sizes, expanded_chunk_sizes def _get_channel_names_for_scene(self, dims: List[str]) -> Optional[List[str]]: # Fast return in None case From c59985e1b8bde1ea6350bf8e39f543f24fc98a96 Mon Sep 17 00:00:00 2001 From: John Russell Date: Thu, 7 Oct 2021 11:18:20 -0400 Subject: [PATCH 21/29] Change name to TiffGlobReader and add some comments continue renaming - tests --- aicsimageio/aics_image.py | 6 +++--- aicsimageio/readers/__init__.py | 4 ++-- .../{glob_reader.py => tiff_glob_reader.py} | 16 +++++++++++++--- aicsimageio/tests/readers/test_glob_reader.py | 16 ++++++++-------- aicsimageio/types.py | 2 +- 5 files changed, 27 insertions(+), 17 deletions(-) rename aicsimageio/readers/{glob_reader.py => tiff_glob_reader.py} (97%) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index 8e94131fe..0445ee4c4 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -15,7 +15,7 @@ from .metadata import utils as metadata_utils from .readers.reader import Reader from .types import PhysicalPixelSizes -from .readers import GlobReader +from .readers import TiffGlobReader ############################################################################### @@ -151,9 +151,9 @@ def determine_reader(image: types.ImageLike, **kwargs: Any) -> Type[Reader]: """ if isinstance(image, list) and isinstance(image[0], str): - return GlobReader + return TiffGlobReader elif isinstance(image, str) and "*" in image: - return GlobReader + return TiffGlobReader # Try reader detection based off of file path extension if isinstance(image, (str, Path)): diff --git a/aicsimageio/readers/__init__.py b/aicsimageio/readers/__init__.py index f70796940..392a91409 100644 --- a/aicsimageio/readers/__init__.py +++ b/aicsimageio/readers/__init__.py @@ -15,7 +15,7 @@ from .ome_tiff_reader import OmeTiffReader # noqa: F401 from .reader import Reader from .tiff_reader import TiffReader # noqa: F401 - from .glob_reader import GlobReader # noqa: F401 + from .tiff_glob_reader import TiffGlobReader # noqa: F401 # add ".relativepath.ClassName" @@ -28,7 +28,7 @@ ".nd2_reader.ND2Reader", ".ome_tiff_reader.OmeTiffReader", ".tiff_reader.TiffReader", - ".glob_reader.GlobReader", + ".tiff_glob_reader.TiffGlobReader", ) _LOOKUP = {k.rsplit(".", 1)[-1]: k for k in _READERS} __all__ = list(_LOOKUP) diff --git a/aicsimageio/readers/glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py similarity index 97% rename from aicsimageio/readers/glob_reader.py rename to aicsimageio/readers/tiff_glob_reader.py index b2a3f1619..ed9c28283 100644 --- a/aicsimageio/readers/glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -32,7 +32,7 @@ TIFF_IMAGE_DESCRIPTION_TAG_INDEX = 270 -class GlobReader(Reader): +class TiffGlobReader(Reader): """ Wraps the tifffile imread API to provide the same aicsimageio Reader API but for @@ -73,10 +73,15 @@ class GlobReader(Reader): single_file_dims : Optional[Tuple] Dimensions that correspond to the data dimensions of a single file in the glob. Default : ('Y', 'X') + + Examples + -------- + reader = TiffGlobReader("~/data/my_dataset/*.tif") + """ @staticmethod - def _is_supported_image(fs: AbstractFileSystem, path: str, **kwargs: Any) -> bool: + def _is_supported_image(fs: AbstractFileSystem, path: types.PathLike, **kwargs: Any) -> bool: try: with fs.open(path) as open_resource: with TiffFile(open_resource): @@ -133,7 +138,11 @@ def __init__( elif isinstance(indexer, pd.DataFrame): self._all_files = indexer self._all_files["filename"] = file_series - + + # If a dim doesn't exist on the file set the column value for that dim to zero. + # If the dim is present, add it to the sort order. Because we are using + # the default dimension ordering, this will naturally create a sort order + # based off the standard dimension order. sort_order = [] for dim in DEFAULT_DIMENSION_ORDER_LIST_WITH_SAMPLES: if dim not in self._all_files.columns and dim not in single_file_dims: @@ -390,6 +399,7 @@ def _read_immediate(self) -> xr.DataArray: arr, dims=dims, coords=coords, + attrs=attrs, ) return x_data diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index cad1cf9e8..674c9130e 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -45,7 +45,7 @@ def make_fake_data_2d(path): def test_glob_reader_2d(tmp_path: Path): reference = make_fake_data_2d(tmp_path) - gr = aicsimageio.readers.GlobReader(str(tmp_path / "2d_images/*.tif")) + gr = aicsimageio.readers.TiffGlobReader(str(tmp_path / "2d_images/*.tif")) assert gr.xarray_dask_data.data.chunksize == (1, 1) + DATA_SHAPE[-3:] @@ -82,14 +82,14 @@ def test_glob_reader_3d(tmp_path: Path): ).astype(int) # do not stack z dimension - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "3d_images/*Z0.tif"), single_file_dims=list("ZYX") ) assert gr.xarray_dask_data.data.chunksize == (1, 1, 2, 7, 8) check_values(gr, reference.isel(Z=slice(0, 2))) # stack along z dimension but do not chunk - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "3d_images/*.tif"), single_file_dims=list("ZYX"), chunk_dims=list("TC"), @@ -98,7 +98,7 @@ def test_glob_reader_3d(tmp_path: Path): check_values(gr, reference) # stack along z and chunk along z - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "3d_images/*.tif"), single_file_dims=list("ZYX") ) assert gr.xarray_dask_data.data.chunksize == (1, 1, 6, 7, 8) @@ -144,21 +144,21 @@ def test_glob_reader_4d(tmp_path: Path): reference = make_fake_data_4d(tmp_path) # stack none - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "4d_images/*T0*Z0.tif"), single_file_dims=list("TZYX") ) assert gr.xarray_dask_data.data.chunksize == (2, 1, 3, 7, 8) check_values(gr, reference.isel(T=slice(0, 2), Z=slice(0, 3))) # stack z and t - chunk z - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "4d_images/*.tif"), single_file_dims=list("TZYX") ) assert gr.xarray_dask_data.data.chunksize == (2, 1, 6, 7, 8) check_values(gr, reference) # stack z and t - chunk z and t - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "4d_images/*.tif"), single_file_dims=list("TZYX"), chunk_dims=["T", "Z"], @@ -167,7 +167,7 @@ def test_glob_reader_4d(tmp_path: Path): check_values(gr, reference) # stack z an t - chunk ztc - gr = aicsimageio.readers.GlobReader( + gr = aicsimageio.readers.TiffGlobReader( str(tmp_path / "4d_images/*.tif"), single_file_dims=list("TZYX"), chunk_dims=list("TCZ"), diff --git a/aicsimageio/types.py b/aicsimageio/types.py index de313eb66..771356875 100644 --- a/aicsimageio/types.py +++ b/aicsimageio/types.py @@ -14,7 +14,7 @@ PathLike = Union[str, Path] ArrayLike = Union[np.ndarray, da.Array] MetaArrayLike = Union[ArrayLike, xr.DataArray] -ImageLike = Union[PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike], List[str]] +ImageLike = Union[PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike], List[PathLike]] # Image Utility Types From b9192b86f6ec31a84a7f8a5e3e52059be83efc2e Mon Sep 17 00:00:00 2001 From: John Russell Date: Thu, 7 Oct 2021 13:25:18 -0400 Subject: [PATCH 22/29] add examples to tiff glob reader docstring --- aicsimageio/readers/tiff_glob_reader.py | 33 ++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index ed9c28283..742992571 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -40,7 +40,7 @@ class TiffGlobReader(Reader): Parameters ---------- - glob_in: Union[str, List[str]] + glob_in: Union[PathLike, List[PathLike]] Glob string that identifies all files to be loaded or a list of paths to the files as returned by glob. indexer: Union[Callable, pandas.DataFrame] @@ -76,12 +76,33 @@ class TiffGlobReader(Reader): Examples -------- - reader = TiffGlobReader("~/data/my_dataset/*.tif") - + # Given files with names like "Position001_t002_c03_z04.tif" + + reader = TiffGlobReader("path/to/data/*.tif") + + # We can use this to read single image tiffs generated by MicroManager + # Micromanager creates directories for each position so we need to recursively glob + # for the images files and pass the list to TiffGlobReader. Note that all images are + # named according to img_channel000_position001_time000000003_z004.tif" + + files = glob.glob("path/to/data/**/*.tif", recursive=True) + + # since the numbers in Micromanager files are not in STCZ order we need + # to use a custom indexer. + + def mm_indexer(path_to_img): + inds = re.findall(r"\d+", Path(path_to_img).name) + series = pd.Series(inds, index=['C', 'S', 'T', 'Z']).astype(int) + return series + + mm_reader = TiffGlobReader(files, indexer=mm_indexer) + """ @staticmethod - def _is_supported_image(fs: AbstractFileSystem, path: types.PathLike, **kwargs: Any) -> bool: + def _is_supported_image( + fs: AbstractFileSystem, path: types.PathLike, **kwargs: Any + ) -> bool: try: with fs.open(path) as open_resource: with TiffFile(open_resource): @@ -92,7 +113,7 @@ def _is_supported_image(fs: AbstractFileSystem, path: types.PathLike, **kwargs: def __init__( self, - glob_in: Union[str, List[str]], + glob_in: Union[types.PathLike, List[types.PathLike]], indexer: Union[pd.DataFrame, Callable] = None, scene_glob_character: str = "S", chunk_dims: Union[str, List[str]] = DEFAULT_CHUNK_DIMS, @@ -138,7 +159,7 @@ def __init__( elif isinstance(indexer, pd.DataFrame): self._all_files = indexer self._all_files["filename"] = file_series - + # If a dim doesn't exist on the file set the column value for that dim to zero. # If the dim is present, add it to the sort order. Because we are using # the default dimension ordering, this will naturally create a sort order From 53d34b4813586d7fa2ee9c2f063e2acc602277c1 Mon Sep 17 00:00:00 2001 From: John Russell Date: Thu, 7 Oct 2021 13:25:38 -0400 Subject: [PATCH 23/29] formatting --- aicsimageio/aics_image.py | 4 ++-- aicsimageio/readers/__init__.py | 1 - aicsimageio/types.py | 4 +++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index 0445ee4c4..fe23600ae 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -149,9 +149,9 @@ def determine_reader(image: types.ImageLike, **kwargs: Any) -> Type[Reader]: exceptions.UnsupportedFileFormatError No reader could be found that supports the provided image. """ - + if isinstance(image, list) and isinstance(image[0], str): - return TiffGlobReader + return TiffGlobReader elif isinstance(image, str) and "*" in image: return TiffGlobReader diff --git a/aicsimageio/readers/__init__.py b/aicsimageio/readers/__init__.py index 392a91409..483dc2bb5 100644 --- a/aicsimageio/readers/__init__.py +++ b/aicsimageio/readers/__init__.py @@ -42,4 +42,3 @@ def __getattr__(name: str) -> Type["Reader"]: mod = import_module(path, __name__) return getattr(mod, clsname) raise AttributeError(f"module {__name__!r} has no attribute import name {name!r}") - diff --git a/aicsimageio/types.py b/aicsimageio/types.py index 771356875..78d76b175 100644 --- a/aicsimageio/types.py +++ b/aicsimageio/types.py @@ -14,7 +14,9 @@ PathLike = Union[str, Path] ArrayLike = Union[np.ndarray, da.Array] MetaArrayLike = Union[ArrayLike, xr.DataArray] -ImageLike = Union[PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike], List[PathLike]] +ImageLike = Union[ + PathLike, ArrayLike, MetaArrayLike, List[MetaArrayLike], List[PathLike] +] # Image Utility Types From 8e1711b5619daa4460ff4101368e521563f11264 Mon Sep 17 00:00:00 2001 From: John Russell Date: Fri, 8 Oct 2021 11:08:23 -0400 Subject: [PATCH 24/29] bump tifffile version to 2021.8.30 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e7710ba42..be04ddb65 100644 --- a/setup.py +++ b/setup.py @@ -92,8 +92,8 @@ def run(self): "lxml>=4.6,<5", "numpy>=1.16,<2", "ome-types>=0.2", - "tifffile>=2021.6.6", "wrapt>=1.12", + "tifffile>=2021.8.30", "xarray>=0.16.1", "xmlschema", # no pin because it's pulled in from OME types "zarr>=2.6,<3", From d73b12ed662b2eda0847e88a5e81cb04c27a7f18 Mon Sep 17 00:00:00 2001 From: John Russell Date: Tue, 12 Oct 2021 16:47:30 -0400 Subject: [PATCH 25/29] Add typing --- aicsimageio/readers/tiff_glob_reader.py | 28 +++++++++---------- aicsimageio/tests/readers/test_glob_reader.py | 22 +++++++++------ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index 742992571..72efd14ed 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -1,17 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import Any, Dict, List, Optional, Tuple, Union, Callable - -import re import glob +import re from collections import OrderedDict from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union + import dask.array as da import numpy as np import pandas as pd import xarray as xr -from dask import delayed from fsspec.spec import AbstractFileSystem from tifffile import TiffFile, TiffFileError, imread from tifffile.tifffile import TiffTags @@ -19,16 +18,15 @@ from .. import constants, exceptions, types from ..dimensions import ( DEFAULT_CHUNK_DIMS, - REQUIRED_CHUNK_DIMS, - DimensionNames, DEFAULT_DIMENSION_ORDER, DEFAULT_DIMENSION_ORDER_LIST_WITH_SAMPLES, + REQUIRED_CHUNK_DIMS, + DimensionNames, ) from ..metadata import utils as metadata_utils from ..utils import io_utils from .reader import Reader - TIFF_IMAGE_DESCRIPTION_TAG_INDEX = 270 @@ -120,7 +118,7 @@ def __init__( dim_order: Optional[Union[List[str], str]] = None, channel_names: Optional[Union[List[str], List[List[str]]]] = None, single_file_shape: Optional[Tuple] = None, - single_file_dims: Optional[Tuple] = ( + single_file_dims: Sequence[str] = ( DimensionNames.SpatialY, DimensionNames.SpatialX, ), @@ -194,6 +192,8 @@ def __init__( f"Number of provided dimension order strings: {len(dim_order)}" ) + self._channel_names = channel_names + # If provided a list if isinstance(channel_names, list): # If provided a list of lists @@ -224,8 +224,6 @@ def __init__( if d in self._all_files.columns or d in self.chunk_dims ) - self._channel_names = channel_names - if single_file_shape is None: with self._fs.open(self._path) as open_resource: with TiffFile(open_resource) as tiff: @@ -345,7 +343,7 @@ def _read_delayed(self) -> xr.DataArray: dims = list(expanded_chunk_sizes.keys()) # Assign dims and coords to construct xarray - channel_names = self._get_channel_names_for_scene(dims) + channel_names = self._get_channel_names_for_scene(dims, d_data.shape) coords = self._get_coords( dims, d_data.shape, self.current_scene_index, channel_names @@ -400,7 +398,7 @@ def _read_immediate(self) -> xr.DataArray: file_dims = [x for x in self._single_file_dims if x not in dims] dims += file_dims - channel_names = self._get_channel_names_for_scene(dims) + channel_names = self._get_channel_names_for_scene(dims, arr.shape) coords = self._get_coords( dims, arr.shape, self.current_scene_index, channel_names @@ -431,7 +429,7 @@ def _get_axes_order( unpack_sizes: OrderedDict, group_sizes: OrderedDict = OrderedDict(), ) -> Tuple: - axes_order = () + axes_order: Tuple[int, ...] = () for d in chunk_sizes: if d in unpack_sizes: axes_order += (list(unpack_sizes.keys()).index(d),) @@ -501,7 +499,9 @@ def _get_expanded_shapes( return expanded_blocks_sizes, expanded_chunk_sizes - def _get_channel_names_for_scene(self, dims: List[str]) -> Optional[List[str]]: + def _get_channel_names_for_scene( + self, dims: List[str], image_shape: Tuple[int, ...] + ) -> Optional[List[str]]: # Fast return in None case if self._channel_names is None: return None diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index 674c9130e..6cf4755dc 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -1,17 +1,21 @@ #! usr/env/bin/python import os import re +from pathlib import Path + import numpy as np -import xarray as xr import pandas as pd import tifffile as tiff +import xarray as xr + import aicsimageio -from pathlib import Path DATA_SHAPE = (3, 4, 5, 6, 7, 8) # STCZYX -def check_values(reader, reference): +def check_values( + reader: aicsimageio.readers.TiffGlobReader, reference: xr.DataArray +) -> None: for i, s in enumerate(reader.scenes): reader.set_scene(s) assert np.all( @@ -20,7 +24,7 @@ def check_values(reader, reference): assert np.all(reference.isel(S=i).data == reader.xarray_data.data) -def make_fake_data_2d(path): +def make_fake_data_2d(path: Path) -> xr.DataArray: data = np.arange(np.prod(DATA_SHAPE), dtype="uint16").reshape(DATA_SHAPE) @@ -43,7 +47,7 @@ def make_fake_data_2d(path): return x_data -def test_glob_reader_2d(tmp_path: Path): +def test_glob_reader_2d(tmp_path: Path) -> None: reference = make_fake_data_2d(tmp_path) gr = aicsimageio.readers.TiffGlobReader(str(tmp_path / "2d_images/*.tif")) @@ -52,7 +56,7 @@ def test_glob_reader_2d(tmp_path: Path): check_values(gr, reference) -def make_fake_data_3d(path): +def make_fake_data_3d(path: Path) -> xr.DataArray: data = np.arange(np.prod(DATA_SHAPE), dtype="uint16").reshape(DATA_SHAPE) @@ -75,7 +79,7 @@ def make_fake_data_3d(path): return x_data -def test_glob_reader_3d(tmp_path: Path): +def test_glob_reader_3d(tmp_path: Path) -> None: reference = make_fake_data_3d(tmp_path) idxr = lambda x: pd.Series( re.findall(r"\d+", Path(x).name), index=list("STC") @@ -105,7 +109,7 @@ def test_glob_reader_3d(tmp_path: Path): check_values(gr, reference) -def make_fake_data_4d(path): +def make_fake_data_4d(path: Path) -> xr.DataArray: data = np.arange(np.prod(DATA_SHAPE), dtype="uint16").reshape(DATA_SHAPE) @@ -140,7 +144,7 @@ def make_fake_data_4d(path): return x_data -def test_glob_reader_4d(tmp_path: Path): +def test_glob_reader_4d(tmp_path: Path) -> None: reference = make_fake_data_4d(tmp_path) # stack none From 372945a8a8552030a1026b13f1502152778feea8 Mon Sep 17 00:00:00 2001 From: John Russell Date: Tue, 12 Oct 2021 17:04:19 -0400 Subject: [PATCH 26/29] flake8 fixes --- aicsimageio/readers/tiff_glob_reader.py | 30 +++++++++++-------- aicsimageio/tests/readers/test_glob_reader.py | 5 ---- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index 72efd14ed..3487f2919 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -32,7 +32,7 @@ class TiffGlobReader(Reader): - """ + r""" Wraps the tifffile imread API to provide the same aicsimageio Reader API but for multifile tiff datasets (and other tifffile supported) images. @@ -42,9 +42,11 @@ class TiffGlobReader(Reader): Glob string that identifies all files to be loaded or a list of paths to the files as returned by glob. indexer: Union[Callable, pandas.DataFrame] - If callable, should consume each filename and return a pd.Series with series index - corresponding to the dimensions and values corresponding to the array index of that image file within the larger array. - Default: None (Look for 4 numbers in the file name and use them as S, T, C, and Z indices. + If callable, should consume each filename and return a pd.Series with series + index corresponding to the dimensions and values corresponding to the array + index of that image file within the larger array. + Default: None (Look for 4 numbers in the file name and use them as + S, T, C, and Z indices.) scene_glob_character: str Character to represent different scenes. Default: "S" @@ -146,10 +148,11 @@ def __init__( # By default we will attempt to parse 4 numbers out of the filename # and assign them in order to be the corresponding S, T, C, and Z indices. - # So indexer("path/to/data/S0_T1_C2_Z3.tif") returns pd.Series([0,1,2,3], index=['S','T','C', 'Z']) - indexer = lambda x: pd.Series( - re.findall(r"\d+", Path(x).name), index=series_idx - ).astype(int) + # So indexer("path/to/data/S0_T1_C2_Z3.tif") returns + # pd.Series([0,1,2,3], index=['S','T','C', 'Z']) + def indexer(x: str) -> pd.Series: + pd.Series(re.findall(r"\d+", Path(x).name), + index=series_idx).astype(int) if callable(indexer): self._all_files = file_series.apply(indexer) @@ -292,7 +295,8 @@ def _read_delayed(self) -> xr.DataArray: # sizes of each chunk chunk_sizes = self._get_chunk_sizes(scene_nunique, group_dims) - # sizes that will be used to reshape the array representing the full glob into separate dimensions + # sizes that will be used to reshape the array representing + # the full glob into separate dimensions. unpack_sizes = OrderedDict( [ (d, s) @@ -304,10 +308,11 @@ def _read_delayed(self) -> xr.DataArray: self._single_file_sizes.values() ) - # after unpacking the result of imread we sometimes need to rearrange dims in case they are in the glob and single files + # after unpacking the result of imread we sometimes need to rearrange dims + # in case they are in the glob and single files. axes_order = self._get_axes_order(chunk_sizes, unpack_sizes, group_sizes) - # expand the sizes with singleton dimensions to facilitate dask.array.block at the end + # expand the sizes with singleton dimensions to facilitate da.block at the end expanded_blocks_sizes, expanded_chunk_sizes = self._get_expanded_shapes( group_sizes, chunk_sizes ) @@ -322,7 +327,8 @@ def _read_delayed(self) -> xr.DataArray: # unpack the first dimension if it contains multiple axes darr = darr.reshape(reshape_sizes) - # Then reorder dimensions so matching ones from the glob and the file are adjacent (glob then file) + # Then reorder dimensions so matching ones from the glob + # and the file are adjacent (glob then file) darr = darr.transpose(axes_order) # Then reshape the array to chunk_sizes diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index 6cf4755dc..edd7c1703 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -1,10 +1,8 @@ #! usr/env/bin/python import os -import re from pathlib import Path import numpy as np -import pandas as pd import tifffile as tiff import xarray as xr @@ -81,9 +79,6 @@ def make_fake_data_3d(path: Path) -> xr.DataArray: def test_glob_reader_3d(tmp_path: Path) -> None: reference = make_fake_data_3d(tmp_path) - idxr = lambda x: pd.Series( - re.findall(r"\d+", Path(x).name), index=list("STC") - ).astype(int) # do not stack z dimension gr = aicsimageio.readers.TiffGlobReader( From f98357ff26e12c7dc9d8315d282245bad06a631c Mon Sep 17 00:00:00 2001 From: John Russell Date: Wed, 13 Oct 2021 12:48:25 -0400 Subject: [PATCH 27/29] pre-commit fixes --- aicsimageio/aics_image.py | 2 +- aicsimageio/readers/__init__.py | 2 +- aicsimageio/readers/tiff_glob_reader.py | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index fe23600ae..93544281e 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -13,9 +13,9 @@ from . import dimensions, exceptions, transforms, types from .formats import FORMAT_IMPLEMENTATIONS, READER_TO_INSTALL from .metadata import utils as metadata_utils +from .readers import TiffGlobReader from .readers.reader import Reader from .types import PhysicalPixelSizes -from .readers import TiffGlobReader ############################################################################### diff --git a/aicsimageio/readers/__init__.py b/aicsimageio/readers/__init__.py index 483dc2bb5..a2ad64150 100644 --- a/aicsimageio/readers/__init__.py +++ b/aicsimageio/readers/__init__.py @@ -14,8 +14,8 @@ from .nd2_reader import ND2Reader # noqa: F401 from .ome_tiff_reader import OmeTiffReader # noqa: F401 from .reader import Reader - from .tiff_reader import TiffReader # noqa: F401 from .tiff_glob_reader import TiffGlobReader # noqa: F401 + from .tiff_reader import TiffReader # noqa: F401 # add ".relativepath.ClassName" diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index 3487f2919..c8d43adc8 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -148,11 +148,12 @@ def __init__( # By default we will attempt to parse 4 numbers out of the filename # and assign them in order to be the corresponding S, T, C, and Z indices. - # So indexer("path/to/data/S0_T1_C2_Z3.tif") returns + # So indexer("path/to/data/S0_T1_C2_Z3.tif") returns # pd.Series([0,1,2,3], index=['S','T','C', 'Z']) def indexer(x: str) -> pd.Series: - pd.Series(re.findall(r"\d+", Path(x).name), - index=series_idx).astype(int) + pd.Series(re.findall(r"\d+", Path(x).name), index=series_idx).astype( + int + ) if callable(indexer): self._all_files = file_series.apply(indexer) @@ -327,7 +328,7 @@ def _read_delayed(self) -> xr.DataArray: # unpack the first dimension if it contains multiple axes darr = darr.reshape(reshape_sizes) - # Then reorder dimensions so matching ones from the glob + # Then reorder dimensions so matching ones from the glob # and the file are adjacent (glob then file) darr = darr.transpose(axes_order) From ad45de4ccc3ac854bbb997c501c9bc81e3fda38e Mon Sep 17 00:00:00 2001 From: John Russell Date: Fri, 15 Oct 2021 13:00:54 -0400 Subject: [PATCH 28/29] Add test for AICSimage Check whether AICSimage correctly determines the reader given a path with a wildcard "*" in it. Also add some catches in aics_image.py and tiff_glob_reader.py for path objects rather than strings. --- aicsimageio/aics_image.py | 2 ++ aicsimageio/readers/tiff_glob_reader.py | 8 +++++--- aicsimageio/tests/readers/test_glob_reader.py | 11 +++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index 93544281e..c2a6269f8 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -154,6 +154,8 @@ def determine_reader(image: types.ImageLike, **kwargs: Any) -> Type[Reader]: return TiffGlobReader elif isinstance(image, str) and "*" in image: return TiffGlobReader + elif isinstance(image, Path) and "*" in str(image): + return TiffGlobReader # Try reader detection based off of file path extension if isinstance(image, (str, Path)): diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index c8d43adc8..de4acd5d9 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -132,6 +132,8 @@ def __init__( file_series = pd.Series(glob.glob(glob_in)) elif isinstance(glob_in, list): file_series = pd.Series(glob_in) + elif isinstance(glob_in, Path) and "*" in str(glob_in): + file_series = pd.Series(glob.glob(str(glob_in))) if len(file_series) == 0: raise ValueError("No files found matching glob pattern") @@ -151,9 +153,9 @@ def __init__( # So indexer("path/to/data/S0_T1_C2_Z3.tif") returns # pd.Series([0,1,2,3], index=['S','T','C', 'Z']) def indexer(x: str) -> pd.Series: - pd.Series(re.findall(r"\d+", Path(x).name), index=series_idx).astype( - int - ) + return pd.Series( + re.findall(r"\d+", Path(x).name), index=series_idx + ).astype(int) if callable(indexer): self._all_files = file_series.apply(indexer) diff --git a/aicsimageio/tests/readers/test_glob_reader.py b/aicsimageio/tests/readers/test_glob_reader.py index edd7c1703..17b0dc848 100644 --- a/aicsimageio/tests/readers/test_glob_reader.py +++ b/aicsimageio/tests/readers/test_glob_reader.py @@ -173,3 +173,14 @@ def test_glob_reader_4d(tmp_path: Path) -> None: ) assert gr.xarray_dask_data.data.chunksize == (4, 5, 6, 7, 8) check_values(gr, reference) + + +def test_aics_image(tmp_path: Path) -> None: + + aicsimage_tiff = aicsimageio.AICSImage(tmp_path / "3d_images/S0_T0_C0_Z0.tif") + assert isinstance(aicsimage_tiff.reader, aicsimageio.readers.tiff_reader.TiffReader) + + aicsimage_tiff_glob = aicsimageio.AICSImage( + tmp_path / "3d_images/*.tif", single_file_dims=list("ZYX") + ) + assert isinstance(aicsimage_tiff_glob.reader, aicsimageio.readers.TiffGlobReader) From 8b9c47c212060895d2ae3a1eb8946e93c2970201 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Mon, 18 Oct 2021 14:02:55 -0400 Subject: [PATCH 29/29] remove other tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 088a156dc..f3ef344ed 100644 --- a/tox.ini +++ b/tox.ini @@ -18,4 +18,4 @@ setenv = extras = test commands = - pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/ {posargs} + pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/test_glob_reader.py {posargs}