From e48512ee6ab5005c6c17cb815c324d357132e8e9 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 23 May 2022 10:50:04 -0400 Subject: [PATCH 01/11] make tests work with either version --- .github/workflows/test-and-lint.yml | 1 + aicsimageio/readers/bioformats_reader.py | 86 ++++++++++++------- .../tests/readers/test_bioformats_reader.py | 13 ++- setup.cfg | 1 + 4 files changed, 67 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 389e1e334..c618f5afa 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -28,6 +28,7 @@ jobs: with: distribution: "temurin" java-version: "11" + cache: 'maven' - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/aicsimageio/readers/bioformats_reader.py b/aicsimageio/readers/bioformats_reader.py index d0f72f863..d66eaa7d9 100644 --- a/aicsimageio/readers/bioformats_reader.py +++ b/aicsimageio/readers/bioformats_reader.py @@ -22,6 +22,7 @@ from .reader import Reader if TYPE_CHECKING: + from bioformats_jar import _loci from fsspec.spec import AbstractFileSystem from .. import types @@ -30,21 +31,27 @@ try: import jpype from bioformats_jar import get_loci -except ImportError: +except ImportError as e: raise ImportError( "bioformats_jar is required for this reader. " - "Install with `pip install bioformats_jar`" - ) + "Install with `pip install bioformats_jar` or `conda install bioformats_jar`" + ) from e +try: + from jgo.jgo import ExecutableNotFound +except ImportError: + + class ExecutableNotFound(Exception): # type: ignore + ... class BioformatsReader(Reader): """Read files using bioformats. This reader requires `bioformats_jar` to be installed in the environment, and - requires the java executable to be available on the path, or via the JAVA_HOME - environment variable. + requires the java executable to be available on the path (or via the JAVA_HOME + environment variable), along with the `mvn` executable. - To install java with conda, run `conda install -c conda-forge openjdk`. + To install java and maven with conda, run `conda install -c conda-forge scyjava`. You may need to deactivate/reactivate your environment after installing. If you are *still* getting a `JVMNotFoundException`, try setting JAVA_HOME as follows: @@ -82,6 +89,7 @@ class BioformatsReader(Reader): tile_size: Optional[Tuple[int, int]] Tuple that sets the tile size of y and x axis, respectively By default, it will use optimal values computed by bioformats itself + Raises ------ exceptions.UnsupportedFileFormatError @@ -131,12 +139,12 @@ def __init__( self._scenes: Tuple[str, ...] = tuple( str(md.getImageName(i)) for i in range(md.getImageCount()) ) - except jpype.JVMNotFoundException: + except RuntimeError: raise - except Exception: + except Exception as e: raise exceptions.UnsupportedFileFormatError( self.__class__.__name__, self._path - ) + ) from e @property def scenes(self) -> Tuple[str, ...]: @@ -196,9 +204,7 @@ def _to_xarray(self, delayed: bool = True) -> xr.DataArray: @staticmethod def bioformats_version() -> str: """The version of the bioformats_package.jar being used.""" - from bioformats_jar import get_loci - - return get_loci().__version__ + return _try_get_loci().__version__ class CoreMeta(NamedTuple): @@ -278,20 +284,7 @@ def __init__( dask_tiles: bool = False, tile_size: Optional[Tuple[int, int]] = None, ): - try: - loci = get_loci() - except jpype.JVMNotFoundException as e: - raise type(e)( - str(e) + "\n\nBioformatsReader requires a java executable to be " - "available in your environment. If you are using conda, you can " - "install with `conda install -c conda-forge openjdk`.\n\n" - "Note: you may need to reactivate your conda environment after " - "installing opendjk. If you still have this error, try:\n\n" - "# mac and linux:\n" - "export JAVA_HOME=$CONDA_PREFIX\n\n" - "# windows:\n" - "set JAVA_HOME=%CONDA_PREFIX%\\Library" - ) + loci = _try_get_loci() # may raise RuntimeError self._path = str(path) self._r = loci.formats.ImageReader() @@ -531,9 +524,7 @@ def _dask_chunk(self, block_id: Tuple[int, ...]) -> np.ndarray: @classmethod def _create_ome_meta(cls) -> Any: """create an OMEXMLMetadata object to populate""" - from bioformats_jar import get_loci - - loci = get_loci() + loci = _try_get_loci() if not cls._service: factory = loci.common.services.ServiceFactory() cls._service = factory.getInstance(loci.formats.services.OMEXMLService) @@ -542,9 +533,7 @@ def _create_ome_meta(cls) -> Any: def _pixtype2dtype(pixeltype: int, little_endian: bool) -> np.dtype: """Convert a loci.formats PixelType integer into a numpy dtype.""" - from bioformats_jar import get_loci - - FT = get_loci().formats.FormatTools + FT = _try_get_loci().formats.FormatTools fmt2type: Dict[int, str] = { FT.INT8: "i1", FT.UINT8: "u1", @@ -615,3 +604,36 @@ def _hide_memoization_warning() -> None: System = jpype.JPackage("java").lang.System System.err.close() + + +MAVEN_ERROR_MSG = """ +BioformatsReader requires the maven ('mvn') executable to be +available in your environment. If you are using conda, you can +install with `conda install -c conda-forge scyjava`. + +Alternatively, install from https://maven.apache.org/download.cgi +""" + +JAVA_ERROR_MSG = """ +BioformatsReader requires a java executable to be available +in your environment. If you are using conda, you can install +with `conda install -c conda-forge scyjava`. + +Note: you may need to reactivate your conda environment after +installing opendjk. If you still have this error, try: + +# mac and linux: +export JAVA_HOME=$CONDA_PREFIX + +# windows: +set JAVA_HOME=%CONDA_PREFIX%\\Library +""" + + +def _try_get_loci() -> _loci.__module_protocol__: + try: + return get_loci() + except ExecutableNotFound as e: + raise RuntimeError(MAVEN_ERROR_MSG) from e + except jpype.JVMNotFoundException as e: + raise RuntimeError(JAVA_ERROR_MSG) from e diff --git a/aicsimageio/tests/readers/test_bioformats_reader.py b/aicsimageio/tests/readers/test_bioformats_reader.py index 5a4a07ef0..3ff2112e1 100644 --- a/aicsimageio/tests/readers/test_bioformats_reader.py +++ b/aicsimageio/tests/readers/test_bioformats_reader.py @@ -16,6 +16,15 @@ from ..conftest import LOCAL, get_resource_full_path, host +try: + bfj = pytest.importorskip("bioformats_jar") + bf_version = tuple(int(x) for x in str(bfj.__version__).split(".")[:2]) +except Exception: + bf_version = (6, 7) + +# bioformats changed their DICOM scene labeling scheme at some point +SERIES_0 = "PRIMARY" if bf_version > (6, 7) else "Series 0" + @host @pytest.mark.parametrize( @@ -209,8 +218,8 @@ ), ( "DICOM_samples_MR-MONO2-8-16x-heart.dcm", - "Series 0", - ("Series 0",), + SERIES_0, + (SERIES_0,), (1, 1, 16, 256, 256), np.uint8, dimensions.DEFAULT_DIMENSION_ORDER, diff --git a/setup.cfg b/setup.cfg index ec37d4980..072516356 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ xfail_strict = true filterwarnings = ignore::UserWarning ignore::FutureWarning + ignore:distutils Version classes are deprecated: addopts = -p no:faulthandler [flake8] From bc142a1e940de41d2e4e53045c8ce2e398482d36 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 23 May 2022 10:56:03 -0400 Subject: [PATCH 02/11] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 124338407..496fb3760 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,10 @@ optionally installed using `[...]` syntax. - For multiple additional supported formats: `pip install aicsimageio[base-imageio,nd2]` - For all additional supported (and openly licensed) formats: `pip install aicsimageio[all]` - Due to the GPL license, LIF support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio readlif>=0.6.4` -- Due to the GPL license, Bio-Formats support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio bioformats_jar` - Due to the GPL license, CZI support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio aicspylibczi>=3.0.5` +- Due to the GPL license, Bio-Formats support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio bioformats_jar`. + + **Important!!** Bio-Formats support also requires a `java` executable in the environment. The simplest method is to install `bioformats_jar` from conda: `conda install -c conda-forge bioformats_jar` (which will additionally bring `openjdk`). ## Documentation From d6856107bb41c73d514f0f04bc256d7e6eb21dee Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 23 May 2022 11:30:54 -0400 Subject: [PATCH 03/11] remove cache --- .github/workflows/test-and-lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index f007acb2e..870defa8d 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -84,7 +84,6 @@ jobs: with: distribution: "temurin" java-version: "11" - cache: 'maven' - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: From a6e6827ee78b32130bec15b2639f3e89d230e229 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 23 May 2022 12:04:11 -0400 Subject: [PATCH 04/11] test from github --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 815ac14ed..959684aaf 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ def run(self): "wheel>=0.34.2", # reader deps *all_formats, - "bioformats_jar", # to test bioformats + "bioformats_jar @ git+ssh://git@github.com/tlambert03/bioformats_jar.git#egg=bioformats_jar", # to test bioformats "bfio>=2.3.0", "readlif>=0.6.4", # to test lif "aicspylibczi>=3.0.5", # to test czi From 6639ffcda92175b256b00f47d39d7ad31437990b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 23 May 2022 14:43:05 -0400 Subject: [PATCH 05/11] remove newline --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 496fb3760..b53c0088e 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,7 @@ optionally installed using `[...]` syntax. - For all additional supported (and openly licensed) formats: `pip install aicsimageio[all]` - Due to the GPL license, LIF support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio readlif>=0.6.4` - Due to the GPL license, CZI support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio aicspylibczi>=3.0.5` -- Due to the GPL license, Bio-Formats support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio bioformats_jar`. - - **Important!!** Bio-Formats support also requires a `java` executable in the environment. The simplest method is to install `bioformats_jar` from conda: `conda install -c conda-forge bioformats_jar` (which will additionally bring `openjdk`). +- Due to the GPL license, Bio-Formats support is not included with the `[all]` extra, and must be installed manually with `pip install aicsimageio bioformats_jar`. **Important!!** Bio-Formats support also requires a `java` executable in the environment. The simplest method is to install `bioformats_jar` from conda: `conda install -c conda-forge bioformats_jar` (which will additionally bring `openjdk`). ## Documentation From 690c4dfc71e15e072c24b8b41a6f888db7f70386 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 24 May 2022 15:29:17 -0400 Subject: [PATCH 06/11] use https --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 959684aaf..4d4c1eb02 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ def run(self): "wheel>=0.34.2", # reader deps *all_formats, - "bioformats_jar @ git+ssh://git@github.com/tlambert03/bioformats_jar.git#egg=bioformats_jar", # to test bioformats + "bioformats_jar @ git+https://git@github.com/tlambert03/bioformats_jar.git#egg=bioformats_jar", # to test bioformats "bfio>=2.3.0", "readlif>=0.6.4", # to test lif "aicspylibczi>=3.0.5", # to test czi From e56961bf51173d06a1b7e5659d56b32e7b7d703b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 24 May 2022 15:30:33 -0400 Subject: [PATCH 07/11] update test and lint --- .github/workflows/test-and-lint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 870defa8d..c97f54fbc 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -1,6 +1,9 @@ name: Test and Lint on: pull_request +permissions: + id-token: write + contents: read # Cancel actions when new commits are pushed to PR concurrency: From 30eb2de01b42b4dcf6a61694f5f937a967b3d5da Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 26 May 2022 13:05:55 -0400 Subject: [PATCH 08/11] remove test-and-lint change --- .github/workflows/test-and-lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 764aaa3da..049828317 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -1,9 +1,6 @@ name: Test and Lint on: pull_request -permissions: - id-token: write - contents: read # Cancel actions when new commits are pushed to PR concurrency: From 8d9ca5c42bfc614395d0718b47b3ec518cb66765 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 26 May 2022 13:35:52 -0400 Subject: [PATCH 09/11] test new bioformats_jar in tox --- setup.py | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index faf4e5d70..649f366d3 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ def run(self): "wheel>=0.34.2", # reader deps *all_formats, - "bioformats_jar @ git+https://git@github.com/tlambert03/bioformats_jar.git#egg=bioformats_jar", # to test bioformats + "bioformats_jar", # to test bioformats "bfio>=2.3.0", "readlif>=0.6.4", # to test lif "aicspylibczi>=3.0.5", # to test czi diff --git a/tox.ini b/tox.ini index b2458a9ef..070c62093 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,8 @@ setenv = PYTHONPATH = {toxinidir} extras = test -deps = bioformats_jar +deps = + bioformats_jar @ git+https://git@github.com/tlambert03/bioformats_jar.git#egg=bioformats_jar commands = pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py {posargs} From 392f90affcc6e47dc63d43367d9193cec77505f7 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 26 May 2022 15:35:02 -0400 Subject: [PATCH 10/11] fix remaining dicom test --- .../tests/readers/extra_readers/test_bioformats_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py b/aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py index 506cbcb41..88edb49c1 100644 --- a/aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py @@ -514,8 +514,8 @@ def test_bioformats_dask_tiling_read(filename: str) -> None: ), ( "DICOM_samples_MR-MONO2-8-16x-heart.dcm", - "Series 0", - ("Series 0",), + SERIES_0, + (SERIES_0,), (1, 1, 16, 256, 256), np.uint8, dimensions.DEFAULT_DIMENSION_ORDER, From 441659476a6312823491ed5d762d89c3a33d8b1a Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 26 May 2022 15:36:39 -0400 Subject: [PATCH 11/11] revert change to tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 070c62093..4e4435d52 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ setenv = extras = test deps = - bioformats_jar @ git+https://git@github.com/tlambert03/bioformats_jar.git#egg=bioformats_jar + bioformats_jar commands = pytest --basetemp={envtmpdir} --cov-report xml --cov-report html --cov=aicsimageio aicsimageio/tests/readers/extra_readers/test_bioformats_reader.py {posargs}