Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion histomics_stream/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@
execution graph.

"""
from . import configure, codecs
from . import configure, codecs # noqa: F401,E402
12 changes: 6 additions & 6 deletions histomics_stream/codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

"""Whole-slide image streamer for machine learning frameworks.

The histomics_stream.codecs module supplies codecs that are useful for Zarr file storage with jpeg or
jpeg2k compression.
The histomics_stream.codecs module supplies codecs that are useful for Zarr file storage
with jpeg or jpeg2k compression.

"""

Expand All @@ -39,10 +39,10 @@ class jpeg(Codec):

Notes
-----
For the code that uses Zarr data storage for jpeg images, we need to supply codecs. Note that we
use this codec instead of that available from the zarr_jpeg package. The latter collapses
dimensions by default, can require us to transpose dimensions, and can miss optimizations based upon
RGB data.
For the code that uses Zarr data storage for jpeg images, we need to supply codecs.
Note that we use this codec instead of that available from the zarr_jpeg package.
The latter collapses dimensions by default, can require us to transpose dimensions,
and can miss optimizations based upon RGB data.

"""

Expand Down
10 changes: 4 additions & 6 deletions histomics_stream/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@

"""Whole-slide image streamer for machine learning frameworks."""

import numpy as np
import torch
from . import configure

"""
See: How to load a list of numpy arrays to pytorch dataset loader?
https://stackoverflow.com/questions/44429199/how-to-load-a-list-of-numpy-arrays-to-pytorch-dataset-loader
Expand All @@ -33,14 +37,8 @@
"""
See: A Comprehensive Guide to the DataLoader Class and Abstractions in PyTorch
https://blog.paperspace.com/dataloaders-abstractions-pytorch/

"""

import numpy as np
import torch

from . import configure


class CreateTorchDataloader(configure.ChunkLocations):
class MyDataset(torch.utils.data.IterableDataset, configure._TilesByCommon):
Expand Down
10 changes: 6 additions & 4 deletions test/test_create_study.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,10 @@ def test_create_study():

# Add a slide to the study, including slide-wide information with it.
my_slide0 = my_slides["Slide_0"] = {}
my_slide0[
"filename"
] = "/tf/notebooks/histomics_stream/example/TCGA-BH-A0BZ-01Z-00-DX1.45EB3E93-A871-49C6-9EAE-90D98AE01913.svs"
my_slide0["filename"] = (
"/tf/notebooks/histomics_stream/example/"
"TCGA-BH-A0BZ-01Z-00-DX1.45EB3E93-A871-49C6-9EAE-90D98AE01913.svs"
)
my_slide0["slide_name"] = "TCGA-BH-A0BZ-01Z-00-DX1"
my_slide0["slide_group"] = "TCGA-BH-A0BZ"
my_slide0["chunk_height"] = 2048
Expand Down Expand Up @@ -81,7 +82,8 @@ def test_create_study():
my_study_by_grid_and_mask,
overlap_height=0,
overlap_width=0,
mask_filename="/tf/notebooks/histomics_stream/example/TCGA-BH-A0BZ-01Z-00-DX1.45EB3E93-A871-49C6-9EAE-90D98AE01913-mask.png",
mask_filename="/tf/notebooks/histomics_stream/example/"
"TCGA-BH-A0BZ-01Z-00-DX1.45EB3E93-A871-49C6-9EAE-90D98AE01913-mask.png",
randomly_select=100,
)
# We could apply this to a subset of the slides, but we will apply it to all
Expand Down
17 changes: 8 additions & 9 deletions test/test_find_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,14 @@
def test_imports_can_be_found():
"""Purpose: Test to check that each import can be found"""

# Import succeeds
import imagecodecs
import itk
import numcodecs
import numpy
import scipy.interpolate
import tensorflow
import torch
import zarr
import imagecodecs # noqa: F401
import itk # noqa: F401
import numcodecs # noqa: F401
import numpy # noqa: F401
import scipy.interpolate # noqa: F401
import tensorflow # noqa: F401
import torch # noqa: F401
import zarr # noqa: F401


if __name__ == "__main__":
Expand Down
21 changes: 17 additions & 4 deletions test/test_mask.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ def test_mask_threshold():

my_study = dict(
version="version-1",
tile_width=5471,
tile_height=5743,
number_pixel_columns_for_tile=5471,
number_pixel_rows_for_tile=5743,
overlap_width=127,
overlap_height=101,
slides=dict(
Expand All @@ -77,15 +77,28 @@ def test_mask_threshold():
tiler_thresholds = (0.00, 0.20, 0.50, 0.80, 1.00)
tilers = [
hs.configure.TilesByGridAndMask(
my_study, mask_filename=mask_path, mask_threshold=threshold
my_study,
mask_filename=mask_path,
mask_threshold=threshold,
number_pixel_overlap_rows_for_tile=101,
number_pixel_overlap_columns_for_tile=127,
)
for threshold in tiler_thresholds
]

def run_tiler(study, tiler):
for slide in study["slides"].values():
tiler(slide)
return tiler.get_tiles(my_study)
return [
(
value["filename"],
[
(tile["tile_top"], tile["tile_left"])
for tile in value["tiles"].values()
],
)
for value in study["slides"].values()
]

found_tiles = [run_tiler(my_study, tiler) for tiler in tilers]

Expand Down