From 9c0f95573b57853d0798439270b12ce5fb4cc204 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 4 Oct 2022 14:17:06 -0500 Subject: [PATCH 01/52] Add conda environment configuration --- environment.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..d7d31481 --- /dev/null +++ b/environment.yml @@ -0,0 +1,17 @@ +# run: conda env create +name: cpcore-dev +channels: + - conda-forge +dependencies: + - boto3 + - docutils + - future + - h5py + - matplotlib + - numpy + - psutil + - pyzmq + - scikit-image + - scipy + - pip: + - centrosome From 2defab5af89ab8dbe63cc13f6eb9461cb34a3ad4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 4 Oct 2022 14:18:09 -0500 Subject: [PATCH 02/52] Remove dependencies on old-style Java-based code --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index fa8c3076..bd61d67f 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,7 @@ "h5py~=3.6.0", "matplotlib>=3.1.3", "numpy>=1.18.2", - "prokaryote==2.4.4", "psutil>=5.7.0", - "python-bioformats==4.0.6", - "python-javabridge==4.0.3", "pyzmq~=22.3", "scikit-image>=0.16.2", "scipy>=1.4.1", From 6bc99e412423f5fc0544b6a1c1ad7054d55b54d7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 4 Oct 2022 14:31:56 -0500 Subject: [PATCH 03/52] Replace javabridge with empty bioformats impl --- cellprofiler_core/analysis/_runner.py | 6 --- cellprofiler_core/bioformats/__init__.py | 3 ++ cellprofiler_core/bioformats/formatreader.py | 23 +++++++++++ cellprofiler_core/bioformats/formatwriter.py | 2 + cellprofiler_core/constants/image.py | 2 +- .../readers/bioformats_reader.py | 2 +- cellprofiler_core/utilities/java.py | 39 ++++--------------- cellprofiler_core/worker/_worker.py | 10 ++--- tests/modules/__init__.py | 4 +- 9 files changed, 42 insertions(+), 49 deletions(-) create mode 100644 cellprofiler_core/bioformats/__init__.py create mode 100644 cellprofiler_core/bioformats/formatreader.py create mode 100644 cellprofiler_core/bioformats/formatwriter.py diff --git a/cellprofiler_core/analysis/_runner.py b/cellprofiler_core/analysis/_runner.py index 9ae20b1d..28740fef 100644 --- a/cellprofiler_core/analysis/_runner.py +++ b/cellprofiler_core/analysis/_runner.py @@ -21,7 +21,6 @@ from . import request as anarequest from ..image import ImageSetList from ..measurement import Measurements -from ..utilities.java import JAVA_STARTED from ..utilities.measurement import load_measurements_from_buffer from ..pipeline import dump from ..preferences import get_plugin_directory @@ -214,8 +213,6 @@ def interface( image_set_end - last image set number to process overwrite - whether to recompute imagesets that already have data in initial_measurements. """ - from javabridge import attach, detach - posted_analysis_started = False acknowledged_thread_start = False measurements = None @@ -426,9 +423,6 @@ def interface( except Exception as e: print(e) finally: - if JAVA_STARTED: - import javabridge - javabridge.detach() # Note - the measurements file is owned by the queue consumer # after this post_event. # diff --git a/cellprofiler_core/bioformats/__init__.py b/cellprofiler_core/bioformats/__init__.py new file mode 100644 index 00000000..3e536e55 --- /dev/null +++ b/cellprofiler_core/bioformats/__init__.py @@ -0,0 +1,3 @@ +READABLE_FORMATS = [] + +PT_UINT16 = None diff --git a/cellprofiler_core/bioformats/formatreader.py b/cellprofiler_core/bioformats/formatreader.py new file mode 100644 index 00000000..553eee82 --- /dev/null +++ b/cellprofiler_core/bioformats/formatreader.py @@ -0,0 +1,23 @@ +K_OMERO_SERVER = None +K_OMERO_PORT = None +K_OMERO_USER = None +K_OMERO_SESSION_ID = None +K_OMERO_CONFIG_FILE = None + +def get_image_reader(): + raise RuntimeError("unimplemented") + +def release_image_reader(): + raise RuntimeError("unimplemented") + +def clear_image_reader_cache(): + raise RuntimeError("unimplemented") + +def set_omero_login_hook(login): + raise RuntimeError("unimplemented") + +def get_omero_credentials(): + raise RuntimeError("unimplemented") + +def use_omero_credentials(credentials): + raise RuntimeError("unimplemented") diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py new file mode 100644 index 00000000..9edebef9 --- /dev/null +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -0,0 +1,2 @@ +def write_image(): + pass diff --git a/cellprofiler_core/constants/image.py b/cellprofiler_core/constants/image.py index 243325a0..c4b4f5f7 100644 --- a/cellprofiler_core/constants/image.py +++ b/cellprofiler_core/constants/image.py @@ -1,4 +1,4 @@ -from bioformats import READABLE_FORMATS +from ..bioformats import READABLE_FORMATS UIC1_TAG = 33628 UIC2_TAG = 33629 diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index d2185da1..bca083b1 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -8,7 +8,7 @@ from ..reader import Reader -from bioformats.formatreader import get_image_reader, release_image_reader, clear_image_reader_cache +from ..bioformats.formatreader import get_image_reader, release_image_reader, clear_image_reader_cache from ..utilities.java import start_java diff --git a/cellprofiler_core/utilities/java.py b/cellprofiler_core/utilities/java.py index 01745c32..09fdcf61 100644 --- a/cellprofiler_core/utilities/java.py +++ b/cellprofiler_core/utilities/java.py @@ -7,21 +7,15 @@ import sys import threading -import javabridge -import prokaryote -from javabridge._javabridge import get_vm - import cellprofiler_core.preferences -JAVA_STARTED = False - LOGGER = logging.getLogger(__name__) def get_jars(): """ - Get the final list of JAR files passed to javabridge + Get the final list of JAR files passed to Java """ class_path = [] @@ -34,26 +28,12 @@ def get_jars(): pathname = os.path.dirname(prokaryote.__file__) - jar_files = [ + return [ os.path.join(pathname, f) for f in os.listdir(pathname) if f.lower().endswith(".jar") ] - class_path += javabridge.JARS + jar_files - - if sys.platform.startswith("win") and not hasattr(sys, "frozen"): - # Have to find tools.jar - from javabridge.locate import find_jdk - - jdk_path = find_jdk() - if jdk_path is not None: - tools_jar = os.path.join(jdk_path, "lib", "tools.jar") - class_path.append(tools_jar) - else: - LOGGER.warning("Failed to find tools.jar") - return class_path - def find_logback_xml(): """Find the location of the logback.xml file for Java logging config @@ -89,9 +69,6 @@ def start_java(): awt from being invoked """ thread_id = threading.get_ident() - if get_vm().is_active(): - javabridge.attach() - return LOGGER.info("Initializing Java Virtual Machine") args = [ "-Dloci.bioformats.loaded=true", @@ -123,18 +100,16 @@ def start_java(): ) % os.environ["CP_JDWP_PORT"] ) - javabridge.start_vm(args=args, class_path=class_path) + #CTR FIXME + #scyjava.start_jvm(args=args, class_path=class_path) # # Enable Bio-Formats directory cacheing # - javabridge.attach() - c_location = javabridge.JClassWrapper("loci.common.Location") - c_location.cacheDirectoryListings(True) + #Location = scyjava.jimport("loci.common.Location") + #Location.cacheDirectoryListings(True) LOGGER.debug("Enabled Bio-formats directory cacheing") - global JAVA_STARTED - JAVA_STARTED = True def stop_java(): LOGGER.info("Shutting down Java Virtual Machine") - javabridge.kill_vm() + #CTR FIXME: scyjava.shutdown_jvm() diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index 6cb61d62..2325b288 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -4,7 +4,6 @@ import time import traceback -import javabridge import zmq from ._pipeline_event_listener import PipelineEventListener @@ -25,7 +24,6 @@ from ..constants.worker import NOTIFY_STOP from ..constants.worker import all_measurements from ..measurement import Measurements -from ..utilities.java import JAVA_STARTED from ..utilities.measurement import load_measurements_from_buffer from ..pipeline import CancelledException from ..preferences import get_awt_headless @@ -40,7 +38,7 @@ class Worker: """ def __init__(self, context, analysis_id, work_request_address, keepalive_address, with_stop_run_loop=True): - from bioformats.formatreader import set_omero_login_hook + from ..bioformats.formatreader import set_omero_login_hook self.context = context self.work_request_address = work_request_address @@ -94,15 +92,13 @@ def __exit__(self, type, value, tb): self.worker.exit_thread() def enter_thread(self): - if not get_awt_headless(): - javabridge.activate_awt() + # CTR FIXME: Do we still need this function? + pass def exit_thread(self): from cellprofiler_core.constants.reader import all_readers for reader in all_readers.values(): reader.clear_cached_readers() - if JAVA_STARTED: - javabridge.detach() def run(self): from cellprofiler_core.pipeline.event import CancelledException diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py index 3fda2a42..43002913 100644 --- a/tests/modules/__init__.py +++ b/tests/modules/__init__.py @@ -3,8 +3,8 @@ import cellprofiler_core.utilities.legacy -from bioformats import PT_UINT16 -from bioformats.formatwriter import write_image +from cellprofiler_core.bioformats import PT_UINT16 +from cellprofiler_core.bioformats.formatwriter import write_image import logging import functools import hashlib From 1a6f61255de0255c5904f7f5b857918dc5ba0181 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 4 Oct 2022 15:31:34 -0500 Subject: [PATCH 04/52] Start implementing Bio-Formats with scyjava Does not work yet. --- cellprofiler_core/bioformats/__init__.py | 24 ++++++++++++- cellprofiler_core/bioformats/formatreader.py | 36 +++++++++++++++++--- cellprofiler_core/bioformats/formatwriter.py | 13 ++++++- cellprofiler_core/bioformats/omexml.py | 3 ++ cellprofiler_core/utilities/java.py | 9 ++--- 5 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 cellprofiler_core/bioformats/omexml.py diff --git a/cellprofiler_core/bioformats/__init__.py b/cellprofiler_core/bioformats/__init__.py index 3e536e55..353a2b2c 100644 --- a/cellprofiler_core/bioformats/__init__.py +++ b/cellprofiler_core/bioformats/__init__.py @@ -1,3 +1,25 @@ -READABLE_FORMATS = [] +# See https://docs.openmicroscopy.org/bio-formats/latest/supported-formats.html +READABLE_FORMATS = ('1sc', '2fl', 'acff', 'afi', 'afm', 'aim', 'al3d', 'ali', + 'am', 'amiramesh', 'apl', 'arf', 'avi', 'bif', 'bin', 'bip', + 'bmp', 'btf', 'c01', 'cfg', 'ch5', 'cif', 'cr2', 'crw', + 'cxd', 'czi', 'dat', 'dcm', 'dib', 'dicom', 'dm2', 'dm3', + 'dm4', 'dti', 'dv', 'eps', 'epsi', 'exp', 'fdf', 'fff', + 'ffr', 'fits', 'flex', 'fli', 'frm', 'gel', 'gif', 'grey', + 'h5', 'hdf', 'hdr', 'hed', 'his', 'htd', 'html', 'hx', 'i2i', + 'ics', 'ids', 'im3', 'img', 'ims', 'inr', 'ipl', 'ipm', 'ipw', + 'j2k', 'jp2', 'jpeg', 'jpf', 'jpg', 'jpk', 'jpx', 'klb', + 'l2d', 'labels', 'lei', 'lif', 'liff', 'lim', 'lms', 'lsm', + 'map', 'mdb', 'mea', 'mnc', 'mng', 'mod', 'mov', 'mrc', 'mrcs', + 'mrw', 'msr', 'mtb', 'mvd2', 'naf', 'nd', 'nd2', 'ndpi', 'ndpis', + 'nef', 'nhdr', 'nii', 'nii.gz', 'nrrd', 'obf', 'obsep', 'oib', + 'oif', 'oir', 'ome', 'ome.btf', 'ome.tf2', 'ome.tf8', 'ome.tif', + 'ome.tiff', 'ome.xml', 'par', 'pbm', 'pcoraw', 'pcx', 'pds', + 'pgm', 'pic', 'pict', 'png', 'pnl', 'ppm', 'pr3', 'ps', 'psd', + 'qptiff', 'r3d', 'raw', 'rcpnl', 'rec', 'res', 'scn', 'sdt', + 'seq', 'sif', 'sld', 'sm2', 'sm3', 'spc', 'spe', 'spi', 'st', + 'stk', 'stp', 'svs', 'sxm', 'tc.', 'tf2', 'tf8', 'tfr', 'tga', + 'tif', 'tiff', 'tnb', 'top', 'txt', 'v', 'vff', 'vms', 'vsi', + 'vws', 'wat', 'wlz', 'wpi', 'xdce', 'xml', 'xqd', 'xqf', 'xv', + 'xys', 'zfp', 'zfr', 'zvi') PT_UINT16 = None diff --git a/cellprofiler_core/bioformats/formatreader.py b/cellprofiler_core/bioformats/formatreader.py index 553eee82..5450a7fe 100644 --- a/cellprofiler_core/bioformats/formatreader.py +++ b/cellprofiler_core/bioformats/formatreader.py @@ -1,20 +1,46 @@ +import logging +import scyjava + +from ..utilities.image import is_file_url +from ..utilities.pathname import url2pathname + +logger = logging.getLogger(__name__) + K_OMERO_SERVER = None K_OMERO_PORT = None K_OMERO_USER = None K_OMERO_SESSION_ID = None K_OMERO_CONFIG_FILE = None -def get_image_reader(): - raise RuntimeError("unimplemented") +scyjava.config.endpoints.append("ome:formats-gpl") + +def get_image_reader(key, path=None, url=None): + '''Make or find an image reader appropriate for the given path + + path - pathname to the reader on disk. + + key - use this key to keep only a single cache member associated with + that key open at a time. + ''' + # E.g.: 6335094544, None, file:///Users/curtis/data/c-elegans-cell-fusion.tif + logger.info("Getting image reader for: %s, %s, %s" % (key, path, url)) + ImageReader = scyjava.jimport("loci.formats.ImageReader") + reader = ImageReader() + if not is_file_url(url): + raise ValueError("Bio-Formats only supports file URLs for the moment") + reader.setId(url2pathname(url)) + return reader def release_image_reader(): raise RuntimeError("unimplemented") def clear_image_reader_cache(): - raise RuntimeError("unimplemented") + pass + #$raise RuntimeError("unimplemented") -def set_omero_login_hook(login): - raise RuntimeError("unimplemented") +def set_omero_login_hook(omero_login): + pass + #raise RuntimeError("unimplemented") def get_omero_credentials(): raise RuntimeError("unimplemented") diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index 9edebef9..c42d20a6 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -1,2 +1,13 @@ -def write_image(): +def write_image( + filename, + pixels, + pixel_type, + c, + z, + t, + size_c, + size_z, + size_t, + channel_names +): pass diff --git a/cellprofiler_core/bioformats/omexml.py b/cellprofiler_core/bioformats/omexml.py new file mode 100644 index 00000000..0e325760 --- /dev/null +++ b/cellprofiler_core/bioformats/omexml.py @@ -0,0 +1,3 @@ +PT_UINT8 = None +PT_UINT16 = None +PT_FLOAT = None diff --git a/cellprofiler_core/utilities/java.py b/cellprofiler_core/utilities/java.py index 09fdcf61..48abe210 100644 --- a/cellprofiler_core/utilities/java.py +++ b/cellprofiler_core/utilities/java.py @@ -26,13 +26,8 @@ def get_jars(): ) LOGGER.debug(" CLASSPATH=" + os.environ["CLASSPATH"]) - pathname = os.path.dirname(prokaryote.__file__) - - return [ - os.path.join(pathname, f) - for f in os.listdir(pathname) - if f.lower().endswith(".jar") - ] + #CTR: FIXME: Return list of all JARs to be added to the classpath. + return class_path def find_logback_xml(): From cc2d162fd6c74c121774eee634c0f9824a59a9e9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 4 Oct 2022 15:56:19 -0500 Subject: [PATCH 05/52] Migrate bioformats fields elsewhere One step toward eliminating the bioformats subpackage. --- cellprofiler_core/bioformats/__init__.py | 26 +------------------- cellprofiler_core/bioformats/omexml.py | 6 ++--- cellprofiler_core/constants/image.py | 30 +++++++++++++++++++++--- tests/modules/__init__.py | 2 +- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cellprofiler_core/bioformats/__init__.py b/cellprofiler_core/bioformats/__init__.py index 353a2b2c..b9cdaa69 100644 --- a/cellprofiler_core/bioformats/__init__.py +++ b/cellprofiler_core/bioformats/__init__.py @@ -1,25 +1 @@ -# See https://docs.openmicroscopy.org/bio-formats/latest/supported-formats.html -READABLE_FORMATS = ('1sc', '2fl', 'acff', 'afi', 'afm', 'aim', 'al3d', 'ali', - 'am', 'amiramesh', 'apl', 'arf', 'avi', 'bif', 'bin', 'bip', - 'bmp', 'btf', 'c01', 'cfg', 'ch5', 'cif', 'cr2', 'crw', - 'cxd', 'czi', 'dat', 'dcm', 'dib', 'dicom', 'dm2', 'dm3', - 'dm4', 'dti', 'dv', 'eps', 'epsi', 'exp', 'fdf', 'fff', - 'ffr', 'fits', 'flex', 'fli', 'frm', 'gel', 'gif', 'grey', - 'h5', 'hdf', 'hdr', 'hed', 'his', 'htd', 'html', 'hx', 'i2i', - 'ics', 'ids', 'im3', 'img', 'ims', 'inr', 'ipl', 'ipm', 'ipw', - 'j2k', 'jp2', 'jpeg', 'jpf', 'jpg', 'jpk', 'jpx', 'klb', - 'l2d', 'labels', 'lei', 'lif', 'liff', 'lim', 'lms', 'lsm', - 'map', 'mdb', 'mea', 'mnc', 'mng', 'mod', 'mov', 'mrc', 'mrcs', - 'mrw', 'msr', 'mtb', 'mvd2', 'naf', 'nd', 'nd2', 'ndpi', 'ndpis', - 'nef', 'nhdr', 'nii', 'nii.gz', 'nrrd', 'obf', 'obsep', 'oib', - 'oif', 'oir', 'ome', 'ome.btf', 'ome.tf2', 'ome.tf8', 'ome.tif', - 'ome.tiff', 'ome.xml', 'par', 'pbm', 'pcoraw', 'pcx', 'pds', - 'pgm', 'pic', 'pict', 'png', 'pnl', 'ppm', 'pr3', 'ps', 'psd', - 'qptiff', 'r3d', 'raw', 'rcpnl', 'rec', 'res', 'scn', 'sdt', - 'seq', 'sif', 'sld', 'sm2', 'sm3', 'spc', 'spe', 'spi', 'st', - 'stk', 'stp', 'svs', 'sxm', 'tc.', 'tf2', 'tf8', 'tfr', 'tga', - 'tif', 'tiff', 'tnb', 'top', 'txt', 'v', 'vff', 'vms', 'vsi', - 'vws', 'wat', 'wlz', 'wpi', 'xdce', 'xml', 'xqd', 'xqf', 'xv', - 'xys', 'zfp', 'zfr', 'zvi') - -PT_UINT16 = None +# NB: No implementation. diff --git a/cellprofiler_core/bioformats/omexml.py b/cellprofiler_core/bioformats/omexml.py index 0e325760..fd1bfd4f 100644 --- a/cellprofiler_core/bioformats/omexml.py +++ b/cellprofiler_core/bioformats/omexml.py @@ -1,3 +1,3 @@ -PT_UINT8 = None -PT_UINT16 = None -PT_FLOAT = None +PT_UINT8 = "uint8" +PT_UINT16 = "uint16" +PT_FLOAT = "float" diff --git a/cellprofiler_core/constants/image.py b/cellprofiler_core/constants/image.py index c4b4f5f7..e4cf1547 100644 --- a/cellprofiler_core/constants/image.py +++ b/cellprofiler_core/constants/image.py @@ -1,5 +1,3 @@ -from ..bioformats import READABLE_FORMATS - UIC1_TAG = 33628 UIC2_TAG = 33629 UIC3_TAG = 33630 @@ -85,6 +83,32 @@ ".zvi", } +# See https://docs.openmicroscopy.org/bio-formats/latest/supported-formats.html +ALL_BIOFORMATS_EXTENSIONS = ( + ".1sc", ".2fl", ".acff", ".afi", ".afm", ".aim", ".al3d", ".ali", + ".am", ".amiramesh", ".apl", ".arf", ".avi", ".bif", ".bin", ".bip", + ".bmp", ".btf", ".c01", ".cfg", ".ch5", ".cif", ".cr2", ".crw", + ".cxd", ".czi", ".dat", ".dcm", ".dib", ".dicom", ".dm2", ".dm3", + ".dm4", ".dti", ".dv", ".eps", ".epsi", ".exp", ".fdf", ".fff", + ".ffr", ".fits", ".flex", ".fli", ".frm", ".gel", ".gif", ".grey", + ".h5", ".hdf", ".hdr", ".hed", ".his", ".htd", ".html", ".hx", ".i2i", + ".ics", ".ids", ".im3", ".img", ".ims", ".inr", ".ipl", ".ipm", ".ipw", + ".j2k", ".jp2", ".jpeg", ".jpf", ".jpg", ".jpk", ".jpx", ".klb", + ".l2d", ".labels", ".lei", ".lif", ".liff", ".lim", ".lms", ".lsm", + ".map", ".mdb", ".mea", ".mnc", ".mng", ".mod", ".mov", ".mrc", ".mrcs", + ".mrw", ".msr", ".mtb", ".mvd2", ".naf", ".nd", ".nd2", ".ndpi", ".ndpis", + ".nef", ".nhdr", ".nii", ".nii.gz", ".nrrd", ".obf", ".obsep", ".oib", + ".oif", ".oir", ".ome", ".ome.btf", ".ome.tf2", ".ome.tf8", ".ome.tif", + ".ome.tiff", ".ome.xml", ".par", ".pbm", ".pcoraw", ".pcx", ".pds", + ".pgm", ".pic", ".pict", ".png", ".pnl", ".ppm", ".pr3", ".ps", ".psd", + ".qptiff", ".r3d", ".raw", ".rcpnl", ".rec", ".res", ".scn", ".sdt", + ".seq", ".sif", ".sld", ".sm2", ".sm3", ".spc", ".spe", ".spi", ".st", + ".stk", ".stp", ".svs", ".sxm", ".tc.", ".tf2", ".tf8", ".tfr", ".tga", + ".tif", ".tiff", ".tnb", ".top", ".txt", ".v", ".vff", ".vms", ".vsi", + ".vws", ".wat", ".wlz", ".wpi", ".xdce", ".xml", ".xqd", ".xqf", ".xv", + ".xys", ".zfp", ".zfr", ".zvi" +) + DISALLOWED_BIOFORMATS_EXTENSIONS = { ".cfg", ".csv", @@ -101,7 +125,7 @@ ".zip" } -BIOFORMATS_IMAGE_EXTENSIONS = set([f".{ext}" for ext in READABLE_FORMATS]) - DISALLOWED_BIOFORMATS_EXTENSIONS +BIOFORMATS_IMAGE_EXTENSIONS = set(ALL_BIOFORMATS_EXTENSIONS) - DISALLOWED_BIOFORMATS_EXTENSIONS ALL_IMAGE_EXTENSIONS = SUPPORTED_IMAGE_EXTENSIONS.union(BIOFORMATS_IMAGE_EXTENSIONS) diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py index 43002913..fac6dc2a 100644 --- a/tests/modules/__init__.py +++ b/tests/modules/__init__.py @@ -3,7 +3,7 @@ import cellprofiler_core.utilities.legacy -from cellprofiler_core.bioformats import PT_UINT16 +from cellprofiler_core.bioformats.omexml import PT_UINT16 from cellprofiler_core.bioformats.formatwriter import write_image import logging import functools From 84ffd52a257ef6c4cace2a50f288f5232917f8b9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 4 Oct 2022 16:24:31 -0500 Subject: [PATCH 06/52] Flesh out the bioformats_reader implementation The python-bioformats read function code has been migrated here now, and updated to use scyjava instead of javabridge. Most code is the same! --- cellprofiler_core/bioformats/__init__.py | 2 +- cellprofiler_core/bioformats/formatreader.py | 30 +-- .../readers/bioformats_reader.py | 222 +++++++++++++++--- 3 files changed, 189 insertions(+), 65 deletions(-) diff --git a/cellprofiler_core/bioformats/__init__.py b/cellprofiler_core/bioformats/__init__.py index b9cdaa69..9f5cf72a 100644 --- a/cellprofiler_core/bioformats/__init__.py +++ b/cellprofiler_core/bioformats/__init__.py @@ -1 +1 @@ -# NB: No implementation. +# NB: No implementation needed. diff --git a/cellprofiler_core/bioformats/formatreader.py b/cellprofiler_core/bioformats/formatreader.py index 5450a7fe..d7c00c92 100644 --- a/cellprofiler_core/bioformats/formatreader.py +++ b/cellprofiler_core/bioformats/formatreader.py @@ -1,10 +1,4 @@ -import logging -import scyjava - -from ..utilities.image import is_file_url -from ..utilities.pathname import url2pathname - -logger = logging.getLogger(__name__) +#CTR FIXME: All of the below should be removed, deleted, perished, annihilated. K_OMERO_SERVER = None K_OMERO_PORT = None @@ -12,28 +6,6 @@ K_OMERO_SESSION_ID = None K_OMERO_CONFIG_FILE = None -scyjava.config.endpoints.append("ome:formats-gpl") - -def get_image_reader(key, path=None, url=None): - '''Make or find an image reader appropriate for the given path - - path - pathname to the reader on disk. - - key - use this key to keep only a single cache member associated with - that key open at a time. - ''' - # E.g.: 6335094544, None, file:///Users/curtis/data/c-elegans-cell-fusion.tif - logger.info("Getting image reader for: %s, %s, %s" % (key, path, url)) - ImageReader = scyjava.jimport("loci.formats.ImageReader") - reader = ImageReader() - if not is_file_url(url): - raise ValueError("Bio-Formats only supports file URLs for the moment") - reader.setId(url2pathname(url)) - return reader - -def release_image_reader(): - raise RuntimeError("unimplemented") - def clear_image_reader_cache(): pass #$raise RuntimeError("unimplemented") diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index bca083b1..50be9c77 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -1,17 +1,25 @@ import collections import itertools +import logging import numpy +import scyjava + +from ..utilities.image import is_file_url +from ..utilities.pathname import url2pathname from ..constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, MD_SIZE_Y, MD_SIZE_X, \ BIOFORMATS_IMAGE_EXTENSIONS from ..reader import Reader -from ..bioformats.formatreader import get_image_reader, release_image_reader, clear_image_reader_cache -from ..utilities.java import start_java +logger = logging.getLogger(__name__) + +# Add Bio-Formats Java dependency. +scyjava.config.endpoints.append("ome:formats-gpl") +logging.info("HELLO! It's a-ME, bioformats_reader. I just added Bio-Formats endpoint.") class BioformatsReader(Reader): """ Derive from this abstract Reader class to create your own image reader in Python @@ -24,16 +32,22 @@ class BioformatsReader(Reader): def __init__(self, image_file): self.variable_revision_number = 1 self._reader = None - start_java() + self._is_file_open = False super().__init__(image_file) def get_reader(self): if self._reader is None: - - self._reader = get_image_reader(id(self), url=self.file.url) + ImageReader = scyjava.jimport("loci.formats.ImageReader") + self._reader = ImageReader() + self._is_file_open = False return self._reader + def _ensure_file_open(self): + if not self._is_file_open: + self._reader.setId(self.file.path) + self._is_file_open = True + def read(self, series=None, index=None, @@ -44,6 +58,7 @@ def read(self, xywh=None, wants_max_intensity=False, channel_names=None, + XYWH=None, ): """Read a single plane from the image file. :param c: read from this channel. `None` = read color image if multichannel @@ -58,18 +73,156 @@ def read(self, :param wants_max_intensity: if `False`, only return the image; if `True`, return a tuple of image and max intensity :param channel_names: provide the channel names for the OME metadata + :param XYWH: a (x, y, w, h) tuple """ + logging.info("--> bioformats_reader.read BEGINS") reader = self.get_reader() - return reader.read( - c=c, - z=z, - t=t, - series=series, - index=index, - rescale=rescale, - wants_max_intensity=wants_max_intensity, - channel_names=channel_names, - ) + self._ensure_file_open() + reader.setSeries(series) + + FormatTools = scyjava.jimport("loci.formats.FormatTools") + ChannelSeparator = scyjava.jimport("loci.formats.ChannelSeparator") + if series is not None: + self._reader.setSeries(series) + + if XYWH is not None: + assert isinstance(XYWH, tuple) and len(XYWH) == 4, "Invalid XYWH tuple" + openBytes_func = lambda x: self._reader.openBytes(x, XYWH[0], XYWH[1], XYWH[2], XYWH[3]) + width, height = XYWH[2], XYWH[3] + else: + openBytes_func = self._reader.openBytes + width, height = self._reader.getSizeX(), self._reader.getSizeY() + + logging.info("--> bioformats_reader.read: Decided openBytes func") + pixel_type = self._reader.getPixelType() + little_endian = self._reader.isLittleEndian() + if pixel_type == FormatTools.INT8: + dtype = np.int8 + scale = 255 + elif pixel_type == FormatTools.UINT8: + dtype = np.uint8 + scale = 255 + elif pixel_type == FormatTools.UINT16: + dtype = 'u2' + scale = 65535 + elif pixel_type == FormatTools.INT16: + dtype = 'i2' + scale = 65535 + elif pixel_type == FormatTools.UINT32: + dtype = 'u4' + scale = 2**32 + elif pixel_type == FormatTools.INT32: + dtype = 'i4' + scale = 2**32-1 + elif pixel_type == FormatTools.FLOAT: + dtype = 'f4' + scale = 1 + elif pixel_type == FormatTools.DOUBLE: + dtype = 'f8' + scale = 1 + max_sample_value = self._reader.getMetadataValue('MaxSampleValue') + if max_sample_value is not None: + try: + scale = jutil.call(max_sample_value, 'intValue', '()I') + except: + logger.warning("WARNING: failed to get MaxSampleValue for image. Intensities may be improperly scaled.") + if index is not None: + logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) index!=None CASE") + image = np.frombuffer(openBytes_func(index), dtype) + if len(image) / height / width in (3,4): + n_channels = int(len(image) / height / width) + if self._reader.isInterleaved(): + image.shape = (height, width, n_channels) + else: + image.shape = (n_channels, height, width) + image = image.transpose(1, 2, 0) + else: + image.shape = (height, width) + elif self._reader.isRGB() and self._reader.isInterleaved(): + index = self._reader.getIndex(z,0,t) + logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGB INTERLEAVED CASE") + image = np.frombuffer(openBytes_func(index), dtype) + image.shape = (height, width, self._reader.getSizeC()) + if image.shape[2] > 3: + image = image[:, :, :3] + elif c is not None and self._reader.getRGBChannelCount() == 1: + index = self._reader.getIndex(z,c,t) + logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGBChannelCount==1 CASE") + image = np.frombuffer(openBytes_func(index), dtype) + image.shape = (height, width) + elif self._reader.getRGBChannelCount() > 1: + logging.info(f"--> bioformats_reader.read: Calling openBytes RGBChannelCount>1 CASE") + n_planes = self._reader.getRGBChannelCount() + rdr = ChannelSeparator(self._reader) + planes = [ + np.frombuffer( + (rdr.openBytes(rdr.getIndex(z,i,t)) if XYWH is None else + rdr.openBytes(rdr.getIndex(z,i,t), XYWH[0], XYWH[1], XYWH[2], XYWH[3])), + dtype + ) for i in range(n_planes)] + + if len(planes) > 3: + planes = planes[:3] + elif len(planes) < 3: + # > 1 and < 3 means must be 2 + # see issue #775 + planes.append(np.zeros(planes[0].shape, planes[0].dtype)) + image = np.dstack(planes) + image.shape=(height, width, 3) + elif self._reader.getSizeC() > 1: + logging.info(f"--> bioformats_reader.read: Calling openBytes SizeC>1 CASE") + images = [ + np.frombuffer(openBytes_func(self._reader.getIndex(z,i,t)), dtype) + for i in range(self._reader.getSizeC())] + image = np.dstack(images) + image.shape = (height, width, self._reader.getSizeC()) + if not channel_names is None: + metadata = metadatatools.MetadataRetrieve(self.metadata) + for i in range(self._reader.getSizeC()): + index = self._reader.getIndex(z, 0, t) + channel_name = metadata.getChannelName(index, i) + if channel_name is None: + channel_name = metadata.getChannelID(index, i) + channel_names.append(channel_name) + elif self._reader.isIndexed(): + # + # The image data is indexes into a color lookup-table + # But sometimes the table is the identity table and just generates + # a monochrome RGB image + # + index = self._reader.getIndex(z,0,t) + logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) INDEXED CASE") + image = np.frombuffer(openBytes_func(index),dtype) + if pixel_type in (FormatTools.INT16, FormatTools.UINT16): + lut = self._reader.get16BitLookupTable() + if lut is not None: + lut = np.array( + [env.get_short_array_elements(d) + for d in env.get_object_array_elements(lut)])\ + .transpose() + else: + lut = self._reader.get8BitLookupTable() + if lut is not None: + lut = np.array( + [env.get_byte_array_elements(d) + for d in env.get_object_array_elements(lut)])\ + .transpose() + image.shape = (height, width) + if (lut is not None) \ + and not np.all(lut == np.arange(lut.shape[0])[:, np.newaxis]): + image = lut[image, :] + else: + index = self._reader.getIndex(z,0,t) + logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) ") + image = np.frombuffer(openBytes_func(index),dtype) + image.shape = (height,width) + + if rescale: + image = image.astype(np.float32) / float(scale) + if wants_max_intensity: + return image, scale + return image + def read_volume(self, series=None, @@ -84,8 +237,8 @@ def read_volume(self, # Whether a volume has planes stored in the z or t axis is often ambiguous. # If z-size > 1 we'll use z, else we'll use t. Otherwise user should choose # an axis to split in Images. - reader = self.get_reader() - bf_reader = reader.rdr + bf_reader = self.get_reader() + self._ensure_file_open() if series is None: series = 0 bf_reader.setSeries(series) @@ -104,7 +257,7 @@ def read_volume(self, t_range = [t] image_stack = [] for z_index, t_index in itertools.product(z_range, t_range): - data = reader.read( + data = self.read( series=series, c=c, z=z_index, @@ -135,30 +288,29 @@ def supports_format(cls, image_file, allow_open=False, volume=False): The volume parameter specifies whether the reader will need to return a 3D array. .""" - file_url = image_file.url.lower() - if file_url.startswith("omero:"): - return 1 - if file_url.endswith(".ome.tif"): - return 2 - if not allow_open: - if image_file.file_extension in BIOFORMATS_IMAGE_EXTENSIONS: - return 3 + if not is_file_url(image_file.url): + return False + try: + logging.info(f"--> bioformats_reader.supports_format: isThisType({image_file.path}, {allow_open}") + ImageReader = scyjava.jimport("loci.formats.ImageReader") + is_this_type = ImageReader().isThisType(image_file.path, allow_open) + logging.info(f"--> bioformats_reader.supports_format: DRUMROLL... is it compatible? ... {is_this_type}") + return 3 if is_this_type else -1 + except Exception as ex: + logger.error(ex) return -1 - else: - try: - get_image_reader(id(image_file), url=image_file.url) - return 3 - except: - return -1 @classmethod def clear_cached_readers(cls): - clear_image_reader_cache() + #CTR FIXME: Do we need this? + #clear_image_reader_cache() + pass def close(self): # If your reader opens a file, this needs to release any active lock, - release_image_reader(id(self)) - self._reader = None + if self._reader is not None: + self._reader.close() + self._reader = None def get_series_metadata(self): """Should return a dictionary with the following keys: From 172d858d7f6e79f65214f52af8200e348f379e12 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 09:56:21 -0500 Subject: [PATCH 07/52] bf_reader: avoid setSeries(None) --- cellprofiler_core/readers/bioformats_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 50be9c77..c62593d5 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -78,7 +78,6 @@ def read(self, logging.info("--> bioformats_reader.read BEGINS") reader = self.get_reader() self._ensure_file_open() - reader.setSeries(series) FormatTools = scyjava.jimport("loci.formats.FormatTools") ChannelSeparator = scyjava.jimport("loci.formats.ChannelSeparator") From df26ef1dab230c4811b020c56b56643866a97892 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 09:58:34 -0500 Subject: [PATCH 08/52] bf_reader: standardize numpy import --- cellprofiler_core/readers/bioformats_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index c62593d5..7b1af75c 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -2,7 +2,7 @@ import itertools import logging -import numpy +import numpy as np import scyjava from ..utilities.image import is_file_url @@ -267,7 +267,7 @@ def read_volume(self, channel_names=channel_names, ) image_stack.append(data) - return numpy.stack(image_stack) + return np.stack(image_stack) @classmethod def supports_format(cls, image_file, allow_open=False, volume=False): From 85efc850876fc3c53dd48c215e6fe29240f5f82d Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 10:25:45 -0500 Subject: [PATCH 09/52] bf_reader: jimport MetadataTools --- cellprofiler_core/readers/bioformats_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 7b1af75c..cc927ee3 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -176,7 +176,7 @@ def read(self, image = np.dstack(images) image.shape = (height, width, self._reader.getSizeC()) if not channel_names is None: - metadata = metadatatools.MetadataRetrieve(self.metadata) + metadata = scyjava.jimport("loci.formats.MetadataTools") for i in range(self._reader.getSizeC()): index = self._reader.getIndex(z, 0, t) channel_name = metadata.getChannelName(index, i) From e60b756a72f3580c093f52a46fbcf98a566500db Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 11:30:09 -0500 Subject: [PATCH 10/52] bf_reader: use scyjava for cast instead of javabridge --- cellprofiler_core/readers/bioformats_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index cc927ee3..a582b53b 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -122,7 +122,7 @@ def read(self, max_sample_value = self._reader.getMetadataValue('MaxSampleValue') if max_sample_value is not None: try: - scale = jutil.call(max_sample_value, 'intValue', '()I') + scale = scyjava.to_python(max_sample_value) except: logger.warning("WARNING: failed to get MaxSampleValue for image. Intensities may be improperly scaled.") if index is not None: From c32bed2fb190c5b553a508dbc332ee4b68247907 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 12:26:31 -0500 Subject: [PATCH 11/52] bf_reader: update lut conversion These are jpype JArrays now and can be iterated directly --- cellprofiler_core/readers/bioformats_reader.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index a582b53b..57a57d6a 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -195,17 +195,11 @@ def read(self, if pixel_type in (FormatTools.INT16, FormatTools.UINT16): lut = self._reader.get16BitLookupTable() if lut is not None: - lut = np.array( - [env.get_short_array_elements(d) - for d in env.get_object_array_elements(lut)])\ - .transpose() + lut = np.array([d for d in lut]).transpose() else: lut = self._reader.get8BitLookupTable() if lut is not None: - lut = np.array( - [env.get_byte_array_elements(d) - for d in env.get_object_array_elements(lut)])\ - .transpose() + lut = np.array([d for d in lut]).transpose() image.shape = (height, width) if (lut is not None) \ and not np.all(lut == np.arange(lut.shape[0])[:, np.newaxis]): From 9e7dbaa29257db6725e232bd2f27050347ea14dd Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 12:31:52 -0500 Subject: [PATCH 12/52] bf_reader: add ChannelFiller TODO --- cellprofiler_core/readers/bioformats_reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 57a57d6a..790200ed 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -184,6 +184,8 @@ def read(self, channel_name = metadata.getChannelID(index, i) channel_names.append(channel_name) elif self._reader.isIndexed(): + # TODO can we use ChannelFiller, which automatically expands channels if they + # are "true" color and does nothing if they are "false" (e.g. applied color table)? # # The image data is indexes into a color lookup-table # But sometimes the table is the identity table and just generates From dea8323a22d73439240e411dbb4ce18d499bf0e7 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 12:39:23 -0500 Subject: [PATCH 13/52] bf_reader: consolidate lut generation --- cellprofiler_core/readers/bioformats_reader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 790200ed..3b3010bb 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -194,14 +194,13 @@ def read(self, index = self._reader.getIndex(z,0,t) logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) INDEXED CASE") image = np.frombuffer(openBytes_func(index),dtype) + lut = None if pixel_type in (FormatTools.INT16, FormatTools.UINT16): lut = self._reader.get16BitLookupTable() - if lut is not None: - lut = np.array([d for d in lut]).transpose() else: lut = self._reader.get8BitLookupTable() - if lut is not None: - lut = np.array([d for d in lut]).transpose() + if lut is not None: + lut = np.array([d for d in lut]).transpose() image.shape = (height, width) if (lut is not None) \ and not np.all(lut == np.arange(lut.shape[0])[:, np.newaxis]): From 624d6faeec49a134bd5a3c8eb17eac123df141a1 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 12:53:56 -0500 Subject: [PATCH 14/52] bf_reader: better define when reader should close Add a shutdown hook to close the reader, and don't try to close it if it's already closed. --- cellprofiler_core/readers/bioformats_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 3b3010bb..a231fd53 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -40,6 +40,7 @@ def get_reader(self): ImageReader = scyjava.jimport("loci.formats.ImageReader") self._reader = ImageReader() self._is_file_open = False + scyjava.when_jvm_stops(lambda: self._reader.close() if self._reader is not None else None) return self._reader @@ -302,7 +303,7 @@ def clear_cached_readers(cls): def close(self): # If your reader opens a file, this needs to release any active lock, - if self._reader is not None: + if self._reader is not None and scyjava.jvm_started(): self._reader.close() self._reader = None From 79dd8b6b27398767b9616bcf4a197f1b943b0e4d Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 14:44:51 -0500 Subject: [PATCH 15/52] Add scyjava to environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index d7d31481..420e5e99 100644 --- a/environment.yml +++ b/environment.yml @@ -13,5 +13,6 @@ dependencies: - pyzmq - scikit-image - scipy + - scyjava - pip: - centrosome From a43fa6a70b50fd48e47814f0f548ec5bada57b4d Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 14:45:08 -0500 Subject: [PATCH 16/52] Add scyjava to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bd61d67f..aae84923 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ "pyzmq~=22.3", "scikit-image>=0.16.2", "scipy>=1.4.1", + "scyjava>=1.6.0", ], license="BSD", name="cellprofiler-core", From c8c175a1a3a555ff9282a99beb0fbe78016dd00f Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 14:56:39 -0500 Subject: [PATCH 17/52] Add scijava-config to jgo endpoints This adds logback-classic and a logback.xml configuration file which sets logging to INFO level, ensuring an slf4j implementation is present and not defaulting to DEBUG level spam. --- cellprofiler_core/readers/bioformats_reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index a231fd53..a44b5934 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -18,6 +18,7 @@ # Add Bio-Formats Java dependency. scyjava.config.endpoints.append("ome:formats-gpl") +scyjava.config.endpoints.append("org.scijava:scijava-config") logging.info("HELLO! It's a-ME, bioformats_reader. I just added Bio-Formats endpoint.") From e405df21a72aa292e912a10635d1a218f929472e Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 15:01:39 -0500 Subject: [PATCH 18/52] bf_reader: remove unused import --- cellprofiler_core/readers/bioformats_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index a44b5934..29be362d 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -6,7 +6,6 @@ import scyjava from ..utilities.image import is_file_url -from ..utilities.pathname import url2pathname from ..constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, MD_SIZE_Y, MD_SIZE_X, \ BIOFORMATS_IMAGE_EXTENSIONS From 706100a7d83908326dfef8ffebb193b0d873afee Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 16:52:38 -0500 Subject: [PATCH 19/52] setup.py: add future module dep This is used and was undeclared, previously coming in transitively through python-bioformats. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index aae84923..e3bde5e5 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ "boto3>=1.12.28", "centrosome==1.2.1", "docutils==0.15.2", + "future>=0.18.2", "h5py~=3.6.0", "matplotlib>=3.1.3", "numpy>=1.18.2", From f34341c5b630ca8f68914a04ce53933712d68f87 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Wed, 5 Oct 2022 16:55:04 -0500 Subject: [PATCH 20/52] bf_reader: fix some reader uses TODO: are these all accurate? --- cellprofiler_core/readers/bioformats_reader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 29be362d..39c98f4b 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -46,7 +46,7 @@ def get_reader(self): def _ensure_file_open(self): if not self._is_file_open: - self._reader.setId(self.file.path) + self.get_reader().setId(self.file.path) self._is_file_open = True def read(self, @@ -77,7 +77,6 @@ def read(self, :param XYWH: a (x, y, w, h) tuple """ logging.info("--> bioformats_reader.read BEGINS") - reader = self.get_reader() self._ensure_file_open() FormatTools = scyjava.jimport("loci.formats.FormatTools") @@ -177,7 +176,7 @@ def read(self, image = np.dstack(images) image.shape = (height, width, self._reader.getSizeC()) if not channel_names is None: - metadata = scyjava.jimport("loci.formats.MetadataTools") + metadata = self._reader.getMetadataStore() for i in range(self._reader.getSizeC()): index = self._reader.getIndex(z, 0, t) channel_name = metadata.getChannelName(index, i) @@ -319,7 +318,8 @@ def get_series_metadata(self): MD_SERIES_NAME - list of series names, one element per series. """ meta_dict = collections.defaultdict(list) - reader = self.get_reader().rdr + self._ensure_file_open() + reader = self.get_reader() series_count = reader.getSeriesCount() meta_dict[MD_SIZE_S] = series_count for i in range(series_count): From 17ab39303f59d2bdf6ad6742e0ada75428c6289e Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 6 Oct 2022 09:50:58 -0500 Subject: [PATCH 21/52] Move jvm startup to utilities/java --- .../readers/bioformats_reader.py | 14 ++-- cellprofiler_core/utilities/java.py | 78 ++++--------------- 2 files changed, 21 insertions(+), 71 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 39c98f4b..05fa5855 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -6,6 +6,7 @@ import scyjava from ..utilities.image import is_file_url +from ..utilities.java import jimport from ..constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, MD_SIZE_Y, MD_SIZE_X, \ BIOFORMATS_IMAGE_EXTENSIONS @@ -15,10 +16,6 @@ logger = logging.getLogger(__name__) -# Add Bio-Formats Java dependency. -scyjava.config.endpoints.append("ome:formats-gpl") -scyjava.config.endpoints.append("org.scijava:scijava-config") - logging.info("HELLO! It's a-ME, bioformats_reader. I just added Bio-Formats endpoint.") class BioformatsReader(Reader): @@ -37,7 +34,7 @@ def __init__(self, image_file): def get_reader(self): if self._reader is None: - ImageReader = scyjava.jimport("loci.formats.ImageReader") + ImageReader = jimport("loci.formats.ImageReader") self._reader = ImageReader() self._is_file_open = False scyjava.when_jvm_stops(lambda: self._reader.close() if self._reader is not None else None) @@ -79,8 +76,8 @@ def read(self, logging.info("--> bioformats_reader.read BEGINS") self._ensure_file_open() - FormatTools = scyjava.jimport("loci.formats.FormatTools") - ChannelSeparator = scyjava.jimport("loci.formats.ChannelSeparator") + FormatTools = jimport("loci.formats.FormatTools") + ChannelSeparator = jimport("loci.formats.ChannelSeparator") if series is not None: self._reader.setSeries(series) @@ -217,7 +214,6 @@ def read(self, return image, scale return image - def read_volume(self, series=None, c=None, @@ -286,7 +282,7 @@ def supports_format(cls, image_file, allow_open=False, volume=False): return False try: logging.info(f"--> bioformats_reader.supports_format: isThisType({image_file.path}, {allow_open}") - ImageReader = scyjava.jimport("loci.formats.ImageReader") + ImageReader = jimport("loci.formats.ImageReader") is_this_type = ImageReader().isThisType(image_file.path, allow_open) logging.info(f"--> bioformats_reader.supports_format: DRUMROLL... is it compatible? ... {is_this_type}") return 3 if is_this_type else -1 diff --git a/cellprofiler_core/utilities/java.py b/cellprofiler_core/utilities/java.py index 48abe210..7d666398 100644 --- a/cellprofiler_core/utilities/java.py +++ b/cellprofiler_core/utilities/java.py @@ -4,55 +4,13 @@ import logging import os -import sys -import threading +import scyjava import cellprofiler_core.preferences LOGGER = logging.getLogger(__name__) - -def get_jars(): - """ - Get the final list of JAR files passed to Java - """ - - class_path = [] - if "CLASSPATH" in os.environ: - class_path += os.environ["CLASSPATH"].split(os.pathsep) - LOGGER.debug( - "Adding Java class path from environment variable, " "CLASSPATH" "" - ) - LOGGER.debug(" CLASSPATH=" + os.environ["CLASSPATH"]) - - #CTR: FIXME: Return list of all JARs to be added to the classpath. - return class_path - - -def find_logback_xml(): - """Find the location of the logback.xml file for Java logging config - - Paths to search are the current directory, the utilities directory - and ../../java/src/main/resources - """ - paths = [ - os.curdir, - os.path.dirname(__file__), - os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), - "java", - "src", - "main", - "resources", - ), - ] - for path in paths: - target = os.path.join(path, "logback.xml") - if os.path.isfile(target): - return target - - def start_java(): """Start CellProfiler's JVM via Javabridge @@ -63,25 +21,18 @@ def start_java(): cpprefs.get_awt_headless() - controls java.awt.headless to prevent awt from being invoked """ - thread_id = threading.get_ident() + if scyjava.jvm_started(): + return + + # Add Bio-Formats Java dependency. + scyjava.config.endpoints.append("ome:formats-gpl") + scyjava.config.endpoints.append("org.scijava:scijava-config") + LOGGER.info("Initializing Java Virtual Machine") args = [ "-Dloci.bioformats.loaded=true", - "-Djava.util.prefs.PreferencesFactory=" - + "org.cellprofiler.headlesspreferences.HeadlessPreferencesFactory", ] - logback_path = find_logback_xml() - - if logback_path is not None: - if sys.platform.startswith("win"): - logback_path = logback_path.replace("\\", "/") - if logback_path[1] == ":": - # \\localhost\x$ is same as x: - logback_path = "//localhost/" + logback_path[0] + "$" + logback_path[2:] - args.append("-Dlogback.configurationFile=%s" % logback_path) - - class_path = get_jars() awt_headless = cellprofiler_core.preferences.get_awt_headless() if awt_headless: LOGGER.debug("JVM will be started with AWT in headless mode") @@ -95,16 +46,19 @@ def start_java(): ) % os.environ["CP_JDWP_PORT"] ) - #CTR FIXME - #scyjava.start_jvm(args=args, class_path=class_path) + scyjava.start_jvm(options=args) # # Enable Bio-Formats directory cacheing # - #Location = scyjava.jimport("loci.common.Location") - #Location.cacheDirectoryListings(True) + Location = scyjava.jimport("loci.common.Location") + Location.cacheDirectoryListings(True) LOGGER.debug("Enabled Bio-formats directory cacheing") def stop_java(): LOGGER.info("Shutting down Java Virtual Machine") - #CTR FIXME: scyjava.shutdown_jvm() + scyjava.shutdown_jvm() + +def jimport(package): + start_java() + return scyjava.jimport(package) From f9751ddf533588025e56b990d45d36d5009abe6b Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 6 Oct 2022 09:51:11 -0500 Subject: [PATCH 22/52] WIP: bfwriter --- cellprofiler_core/bioformats/formatwriter.py | 38 +++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index c42d20a6..38f00c1d 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -1,3 +1,5 @@ +import scyjava + def write_image( filename, pixels, @@ -10,4 +12,38 @@ def write_image( size_t, channel_names ): - pass + omexml = scyjava.jimport() + omexml.image(0).Name = os.path.split(pathname)[1] + p = omexml.image(0).Pixels + assert isinstance(p, ome.OMEXML.Pixels) + p.SizeX = pixels.shape[1] + p.SizeY = pixels.shape[0] + p.SizeC = size_c + p.SizeT = size_t + p.SizeZ = size_z + p.DimensionOrder = ome.DO_XYCZT + p.PixelType = pixel_type + index = c + size_c * z + size_c * size_z * t + if pixels.ndim == 3: + p.SizeC = pixels.shape[2] + p.Channel(0).SamplesPerPixel = pixels.shape[2] + omexml.structured_annotations.add_original_metadata( + ome.OM_SAMPLES_PER_PIXEL, str(pixels.shape[2])) + elif size_c > 1: + p.channel_count = size_c + + pixel_buffer = convert_pixels_to_buffer(pixels, pixel_type) + xml = omexml.to_xml() + script = """ + importClass(Packages.loci.formats.services.OMEXMLService, + Packages.loci.common.services.ServiceFactory, + Packages.loci.formats.ImageWriter); + var service = new ServiceFactory().getInstance(OMEXMLService); + var metadata = service.createOMEXMLMetadata(xml); + var writer = new ImageWriter(); + writer.setMetadataRetrieve(metadata); + writer.setId(path); + writer.setInterleaved(true); + writer.saveBytes(index, buffer); + writer.close(); + """ From 5ae878541ab76184ea4ef208f0d1b589296db435 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 6 Oct 2022 11:11:40 -0500 Subject: [PATCH 23/52] Add fixme: buffer conversion --- cellprofiler_core/readers/bioformats_reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 05fa5855..1510d13f 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -90,6 +90,7 @@ def read(self, width, height = self._reader.getSizeX(), self._reader.getSizeY() logging.info("--> bioformats_reader.read: Decided openBytes func") + # FIXME instead of np.frombuffer use scyjava.to_python, ideally that wraps memory pixel_type = self._reader.getPixelType() little_endian = self._reader.isLittleEndian() if pixel_type == FormatTools.INT8: From f1b8863ad5c80e89c7e963ed30cb2e2c17b56920 Mon Sep 17 00:00:00 2001 From: Nodari Gogoberidze Date: Thu, 6 Oct 2022 15:38:21 -0500 Subject: [PATCH 24/52] [wip] Build out most of writer --- cellprofiler_core/bioformats/formatwriter.py | 108 +++++++++++++------ 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index 38f00c1d..f07bb819 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -1,7 +1,17 @@ +import os import scyjava +import numpy as np +import logging + +from . import omexml +from ..utilities.java import jimport + +logger = logging.getLogger(__name__) + +p2j = lambda v: scyjava.to_java(v) def write_image( - filename, + pathname, pixels, pixel_type, c, @@ -12,38 +22,72 @@ def write_image( size_t, channel_names ): - omexml = scyjava.jimport() - omexml.image(0).Name = os.path.split(pathname)[1] - p = omexml.image(0).Pixels - assert isinstance(p, ome.OMEXML.Pixels) - p.SizeX = pixels.shape[1] - p.SizeY = pixels.shape[0] - p.SizeC = size_c - p.SizeT = size_t - p.SizeZ = size_z - p.DimensionOrder = ome.DO_XYCZT - p.PixelType = pixel_type - index = c + size_c * z + size_c * size_z * t + ImageWriter = jimport("loci.formats.ImageWriter") + OMEXMLService = jimport("loci.formats.services.OMEXMLServiceImpl") + DimensionsOrder = jimport("ome.xml.model.enums.DimensionOrder") + PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger") + PixelType = jimport("ome.xml.model.enums.PixelType") + + omexml_service = OMEXMLService() + metadata = omexml_service.createOMEXMLMetadata() + metadata.createRoot() + + metadata.setImageName(os.path.split(pathname)[1], 0) + metadata.setPixelsSizeX(PositiveInteger(p2j(pixels.shape[1])), 0) + metadata.setPixelsSizeY(PositiveInteger(p2j(pixels.shape[0])), 0) + metadata.setPixelsSizeC(PositiveInteger(p2j(size_c)), 0) + metadata.setPixelsSizeZ(PositiveInteger(p2j(size_z)), 0) + metadata.setPixelsSizeT(PositiveInteger(p2j(size_t)), 0) + metadata.setPixelsDimensionOrder(DimensionsOrder.XYCTZ, 0) + metadata.setPixelsType(PixelType.fromString(pixel_type), 0) + + if pixels.ndim == 3: - p.SizeC = pixels.shape[2] - p.Channel(0).SamplesPerPixel = pixels.shape[2] - omexml.structured_annotations.add_original_metadata( - ome.OM_SAMPLES_PER_PIXEL, str(pixels.shape[2])) + metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0) + metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, 0) + omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2])) + # omexml.structured_annotations.add_original_metadata( + # ome.OM_SAMPLES_PER_PIXEL, str(pixels.shape[2])) elif size_c > 1: - p.channel_count = size_c + # meta.channel_count = size_c <- cant find + metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0) + metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, 0) + omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2])) + + metadata.setImageID("Image:0", 0) + metadata.setPixelsID("Pixels:0", 0) + for i in range(size_c): + metadata.setChannelID(f"Channel:0:{i}", 0, i) + + index = c + size_c * z + size_c * size_z * t pixel_buffer = convert_pixels_to_buffer(pixels, pixel_type) - xml = omexml.to_xml() - script = """ - importClass(Packages.loci.formats.services.OMEXMLService, - Packages.loci.common.services.ServiceFactory, - Packages.loci.formats.ImageWriter); - var service = new ServiceFactory().getInstance(OMEXMLService); - var metadata = service.createOMEXMLMetadata(xml); - var writer = new ImageWriter(); - writer.setMetadataRetrieve(metadata); - writer.setId(path); - writer.setInterleaved(true); - writer.saveBytes(index, buffer); - writer.close(); - """ + + + writer = ImageWriter() + writer.setMetadataRetrieve(metadata) + writer.setId(pathname) + writer.setInterleaved(True) + writer.saveBytes(index, pixel_buffer) + writer.close() + +def convert_pixels_to_buffer(pixels, pixel_type): + '''Convert the pixels in the image into a buffer of the right pixel type + + pixels - a 2d monochrome or color image + + pixel_type - one of the OME pixel types + + returns a 1-d byte array + ''' + if pixel_type == omexml.PT_UINT8: + as_dtype = np.uint8 + elif pixel_type == omexml.PT_UINT16: + as_dtype = " Date: Thu, 6 Oct 2022 16:10:27 -0500 Subject: [PATCH 25/52] formatwriter: get omexmlservice from factory --- cellprofiler_core/bioformats/formatwriter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index f07bb819..a6388e69 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -23,12 +23,13 @@ def write_image( channel_names ): ImageWriter = jimport("loci.formats.ImageWriter") - OMEXMLService = jimport("loci.formats.services.OMEXMLServiceImpl") + OMEXMLServiceFactory = jimport("loci.common.services.ServiceFactory") DimensionsOrder = jimport("ome.xml.model.enums.DimensionOrder") PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger") PixelType = jimport("ome.xml.model.enums.PixelType") + OMEXMLService = jimport("loci.formats.services.OMEXMLService") - omexml_service = OMEXMLService() + omexml_service = OMEXMLServiceFactory().getInstance(OMEXMLService) metadata = omexml_service.createOMEXMLMetadata() metadata.createRoot() From a9bffb9b55794f57ffa9e2ce954188b5778d14b7 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 6 Oct 2022 16:13:49 -0500 Subject: [PATCH 26/52] WIP formatwriter fixme --- cellprofiler_core/bioformats/formatwriter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index a6388e69..ad8d5fc5 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -52,6 +52,7 @@ def write_image( elif size_c > 1: # meta.channel_count = size_c <- cant find metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0) + # FIXME do this per channel metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, 0) omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2])) @@ -60,6 +61,7 @@ def write_image( for i in range(size_c): metadata.setChannelID(f"Channel:0:{i}", 0, i) + metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, i) index = c + size_c * z + size_c * size_z * t pixel_buffer = convert_pixels_to_buffer(pixels, pixel_type) From a22b2c0040f853c1889e08fd1877cf0fd0be4d3e Mon Sep 17 00:00:00 2001 From: Nodari Gogoberidze Date: Thu, 6 Oct 2022 16:50:55 -0500 Subject: [PATCH 27/52] Fix writer writer now works on writing single channel images, but wip on multi-channel --- cellprofiler_core/bioformats/formatwriter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index ad8d5fc5..aeace308 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -28,6 +28,7 @@ def write_image( PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger") PixelType = jimport("ome.xml.model.enums.PixelType") OMEXMLService = jimport("loci.formats.services.OMEXMLService") + IMetadata = jimport("loci.formats.meta.IMetadata") omexml_service = OMEXMLServiceFactory().getInstance(OMEXMLService) metadata = omexml_service.createOMEXMLMetadata() @@ -39,6 +40,7 @@ def write_image( metadata.setPixelsSizeC(PositiveInteger(p2j(size_c)), 0) metadata.setPixelsSizeZ(PositiveInteger(p2j(size_z)), 0) metadata.setPixelsSizeT(PositiveInteger(p2j(size_t)), 0) + metadata.setPixelsBinDataBigEndian(True, 0, 0) metadata.setPixelsDimensionOrder(DimensionsOrder.XYCTZ, 0) metadata.setPixelsType(PixelType.fromString(pixel_type), 0) @@ -61,7 +63,7 @@ def write_image( for i in range(size_c): metadata.setChannelID(f"Channel:0:{i}", 0, i) - metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, i) + metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(1)), 0, i) index = c + size_c * z + size_c * size_z * t pixel_buffer = convert_pixels_to_buffer(pixels, pixel_type) From f07df8442431a4d376553c8793cbe375bb7aae1b Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Tue, 29 Nov 2022 19:44:55 -0500 Subject: [PATCH 28/52] Refactor logging add log level and origin to log messages --- .../readers/bioformats_reader.py | 30 +++++++++---------- cellprofiler_core/utilities/hdf5_dict.py | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 8782c70d..fd0ab2f5 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -14,9 +14,9 @@ from ..reader import Reader -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) -logger.info("HELLO! It's a-ME, bioformats_reader. I just added Bio-Formats endpoint.") +LOGGER.info("HELLO! It's a-ME, bioformats_reader. I just added Bio-Formats endpoint.") # bioformats returns 2 for these, imageio reader returns 3 SUPPORTED_EXTENSIONS = {'.tiff', '.tif', '.ome.tif', '.ome.tiff'} @@ -81,7 +81,7 @@ def read(self, :param channel_names: provide the channel names for the OME metadata :param XYWH: a (x, y, w, h) tuple """ - logging.info("--> bioformats_reader.read BEGINS") + LOGGER.info("--> bioformats_reader.read BEGINS") self._ensure_file_open() FormatTools = jimport("loci.formats.FormatTools") @@ -97,7 +97,7 @@ def read(self, openBytes_func = self._reader.openBytes width, height = self._reader.getSizeX(), self._reader.getSizeY() - logging.info("--> bioformats_reader.read: Decided openBytes func") + LOGGER.info("--> bioformats_reader.read: Decided openBytes func") # FIXME instead of np.frombuffer use scyjava.to_python, ideally that wraps memory pixel_type = self._reader.getPixelType() little_endian = self._reader.isLittleEndian() @@ -130,9 +130,9 @@ def read(self, try: scale = scyjava.to_python(max_sample_value) except: - logger.warning("WARNING: failed to get MaxSampleValue for image. Intensities may be improperly scaled.") + LOGGER.warning("WARNING: failed to get MaxSampleValue for image. Intensities may be improperly scaled.") if index is not None: - logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) index!=None CASE") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) index!=None CASE") image = np.frombuffer(openBytes_func(index), dtype) if len(image) / height / width in (3,4): n_channels = int(len(image) / height / width) @@ -145,18 +145,18 @@ def read(self, image.shape = (height, width) elif self._reader.isRGB() and self._reader.isInterleaved(): index = self._reader.getIndex(z,0,t) - logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGB INTERLEAVED CASE") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGB INTERLEAVED CASE") image = np.frombuffer(openBytes_func(index), dtype) image.shape = (height, width, self._reader.getSizeC()) if image.shape[2] > 3: image = image[:, :, :3] elif c is not None and self._reader.getRGBChannelCount() == 1: index = self._reader.getIndex(z,c,t) - logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGBChannelCount==1 CASE") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGBChannelCount==1 CASE") image = np.frombuffer(openBytes_func(index), dtype) image.shape = (height, width) elif self._reader.getRGBChannelCount() > 1: - logging.info(f"--> bioformats_reader.read: Calling openBytes RGBChannelCount>1 CASE") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes RGBChannelCount>1 CASE") n_planes = self._reader.getRGBChannelCount() rdr = ChannelSeparator(self._reader) planes = [ @@ -175,7 +175,7 @@ def read(self, image = np.dstack(planes) image.shape=(height, width, 3) elif self._reader.getSizeC() > 1: - logging.info(f"--> bioformats_reader.read: Calling openBytes SizeC>1 CASE") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes SizeC>1 CASE") images = [ np.frombuffer(openBytes_func(self._reader.getIndex(z,i,t)), dtype) for i in range(self._reader.getSizeC())] @@ -198,7 +198,7 @@ def read(self, # a monochrome RGB image # index = self._reader.getIndex(z,0,t) - logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) INDEXED CASE") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) INDEXED CASE") image = np.frombuffer(openBytes_func(index),dtype) lut = None if pixel_type in (FormatTools.INT16, FormatTools.UINT16): @@ -213,7 +213,7 @@ def read(self, image = lut[image, :] else: index = self._reader.getIndex(z,0,t) - logging.info(f"--> bioformats_reader.read: Calling openBytes({index}) ") + LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) ") image = np.frombuffer(openBytes_func(index),dtype) image.shape = (height,width) @@ -288,12 +288,12 @@ def supports_format(cls, image_file, allow_open=False, volume=False): The volume parameter specifies whether the reader will need to return a 3D array. .""" try: - logging.info(f"--> bioformats_reader.supports_format: isThisType({image_file.path}, {allow_open}") + LOGGER.info(f"--> bioformats_reader.supports_format: isThisType({image_file.path}, allow_open={allow_open})") ImageReader = jimport("loci.formats.ImageReader") is_this_type = ImageReader().isThisType(image_file.path, allow_open) - logging.info(f"--> bioformats_reader.supports_format: DRUMROLL... is it compatible? ... {is_this_type}") + LOGGER.info(f"--> bioformats_reader.supports_format: DRUMROLL... is it compatible? ... {is_this_type}") except Exception as ex: - logger.error(ex) + LOGGER.error(ex) if image_file.scheme not in SUPPORTED_SCHEMES: return -1 diff --git a/cellprofiler_core/utilities/hdf5_dict.py b/cellprofiler_core/utilities/hdf5_dict.py index d79cac8a..a682aaa8 100644 --- a/cellprofiler_core/utilities/hdf5_dict.py +++ b/cellprofiler_core/utilities/hdf5_dict.py @@ -183,7 +183,7 @@ def __init__( self.filename = hdf5_filename self.top_level_group_name = top_level_group_name LOGGER.debug( - "HDF5Dict.__init__(): %s, temporary=%s, copy=%s, mode=%s", + "%s, temporary=%s, copy=%s, mode=%s", self.filename, self.is_temporary, copy, From d214b96a5442910fe9795015a40f4725eaa71243 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Wed, 30 Nov 2022 14:37:44 -0500 Subject: [PATCH 29/52] Refactor logging use named logger everywhere --- cellprofiler_core/analysis/_runner.py | 48 ++++---- cellprofiler_core/bioformats/formatwriter.py | 3 +- cellprofiler_core/image/_image_set.py | 4 +- .../image/abstract_image/_abstract_image.py | 4 +- .../image/abstract_image/file/_file_image.py | 2 +- .../measurement/_measurements.py | 10 +- cellprofiler_core/module/_module.py | 4 +- cellprofiler_core/modules/images.py | 7 +- cellprofiler_core/modules/metadata.py | 7 +- cellprofiler_core/modules/namesandtypes.py | 7 +- cellprofiler_core/pipeline/_image_file.py | 12 +- cellprofiler_core/pipeline/_image_plane.py | 4 +- cellprofiler_core/pipeline/_pipeline.py | 51 ++++---- cellprofiler_core/pipeline/io/_v6.py | 4 +- cellprofiler_core/preferences/__init__.py | 19 +-- cellprofiler_core/readers/ngff_reader.py | 6 +- cellprofiler_core/setting/_image_plane.py | 4 +- cellprofiler_core/setting/range/_range.py | 4 +- .../setting/text/number/_number.py | 4 +- .../utilities/core/modules/__init__.py | 10 +- cellprofiler_core/utilities/hdf5_dict.py | 4 +- cellprofiler_core/utilities/zmq/__init__.py | 15 ++- cellprofiler_core/utilities/zmq/_boundary.py | 14 ++- cellprofiler_core/worker/__init__.py | 17 +-- cellprofiler_core/worker/_worker.py | 38 +++--- cellprofiler_core/workspace/_workspace.py | 6 +- tests/analysis/test_analysis.py | 109 +++++++++--------- tests/modules/__init__.py | 17 ++- tests/workspace/test_workspace.py | 4 +- 29 files changed, 241 insertions(+), 197 deletions(-) diff --git a/cellprofiler_core/analysis/_runner.py b/cellprofiler_core/analysis/_runner.py index 28740fef..68dd40db 100644 --- a/cellprofiler_core/analysis/_runner.py +++ b/cellprofiler_core/analysis/_runner.py @@ -37,6 +37,8 @@ from ..workspace import Workspace +LOGGER = logging.getLogger(__name__) + class Runner: """The Runner manages two threads (per instance) and all of the workers (per class, i.e., singletons). @@ -166,22 +168,22 @@ def notify_threads(self): def cancel(self): """cancel the analysis run""" - logging.debug("Stopping workers") + LOGGER.debug("Stopping workers") self.stop_workers() - logging.debug("Canceling run") + LOGGER.debug("Canceling run") self.cancelled = True self.paused = False self.notify_threads() - logging.debug("Waiting on interface thread") + LOGGER.debug("Waiting on interface thread") self.interface_thread.join() - logging.debug("Waiting on jobserver thread") + LOGGER.debug("Waiting on jobserver thread") self.jobserver_thread.join() self.interface_thread = None self.jobserver_thread = None self.work_queue = queue.Queue() self.in_process_queue = queue.Queue() self.finished_queue = queue.Queue() - logging.debug("Cancel complete") + LOGGER.debug("Cancel complete") def pause(self): """pause the analysis run""" @@ -523,21 +525,21 @@ def jobserver(self, start_signal): continue if isinstance(req, anarequest.PipelinePreferences): - logging.debug("Received pipeline preferences request") + LOGGER.debug("Received pipeline preferences request") req.reply( Reply( pipeline_blob=numpy.array(self.pipeline_as_string()), preferences=preferences_as_dict(), ) ) - logging.debug("Replied to pipeline preferences request") + LOGGER.debug("Replied to pipeline preferences request") elif isinstance(req, anarequest.InitialMeasurements): - logging.debug("Received initial measurements request") + LOGGER.debug("Received initial measurements request") req.reply(Reply(buf=self.initial_measurements_buf)) - logging.debug("Replied to initial measurements request") + LOGGER.debug("Replied to initial measurements request") elif isinstance(req, anarequest.Work): if not self.work_queue.empty(): - logging.debug("Received work request") + LOGGER.debug("Received work request") ( job, worker_runs_post_group, @@ -551,7 +553,7 @@ def jobserver(self, start_signal): ) ) self.queue_dispatched_job(job) - logging.debug( + LOGGER.debug( "Dispatched job: image sets=%s" % ",".join([str(i) for i in job]) ) @@ -562,21 +564,21 @@ def jobserver(self, start_signal): elif isinstance(req, anareply.ImageSetSuccess): # interface() is responsible for replying, to allow it to # request the shared_state dictionary if needed. - logging.debug("Received ImageSetSuccess") + LOGGER.debug("Received ImageSetSuccess") self.queue_imageset_finished(req) - logging.debug("Enqueued ImageSetSuccess") + LOGGER.debug("Enqueued ImageSetSuccess") elif isinstance(req, anarequest.SharedDictionary): - logging.debug("Received shared dictionary request") + LOGGER.debug("Received shared dictionary request") req.reply(anareply.SharedDictionary(dictionaries=self.shared_dicts)) - logging.debug("Sent shared dictionary reply") + LOGGER.debug("Sent shared dictionary reply") elif isinstance(req, anarequest.MeasurementsReport): - logging.debug("Received measurements report") + LOGGER.debug("Received measurements report") self.queue_received_measurements(req.image_set_numbers, req.buf) req.reply(anareply.Ack()) - logging.debug("Acknowledged measurements report") + LOGGER.debug("Acknowledged measurements report") elif isinstance(req, anarequest.AnalysisCancel): # Signal the interface that we are cancelling - logging.debug("Received analysis worker cancel request") + LOGGER.debug("Received analysis worker cancel request") with self.interface_work_cv: self.cancelled = True self.interface_work_cv.notify() @@ -593,13 +595,13 @@ def jobserver(self, start_signal): anarequest.OmeroLogin, ), ): - logging.debug("Enqueueing interactive request") + LOGGER.debug("Enqueueing interactive request") # bump upward self.post_event(req) - logging.debug("Interactive request enqueued") + LOGGER.debug("Interactive request enqueued") else: msg = "Unknown request from worker: %s of type %s" % (req, type(req)) - logging.error(msg) + LOGGER.error(msg) raise ValueError(msg) # stop the ZMQ-boundary thread - will also deal with any requests waiting on replies @@ -641,7 +643,7 @@ def start_workers(self, num=None): boundary = self.boundary - logging.info("Starting workers on address %s" % boundary.request_address) + LOGGER.info("Starting workers on address %s" % boundary.request_address) close_fds = False @@ -701,7 +703,7 @@ def run_logger(workR, widx): line = line.decode("utf-8") if not line: break - logging.info("Worker %d: %s", widx, line.rstrip()) + LOGGER.info("Worker %d: %s", widx, line.rstrip()) except: break diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index aeace308..757f1f86 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -6,7 +6,8 @@ from . import omexml from ..utilities.java import jimport -logger = logging.getLogger(__name__) + +LOGGER = logging.getLogger(__name__) p2j = lambda v: scyjava.to_java(v) diff --git a/cellprofiler_core/image/_image_set.py b/cellprofiler_core/image/_image_set.py index e8cacda3..08991933 100644 --- a/cellprofiler_core/image/_image_set.py +++ b/cellprofiler_core/image/_image_set.py @@ -7,6 +7,8 @@ from .abstract_image import VanillaImage +LOGGER = logging.getLogger(__name__) + class ImageSet: """Represents the images for a particular iteration of a pipeline @@ -77,7 +79,7 @@ def get_image( ) if image.pixel_data.shape[-1] == 4: - logging.warning("Discarding alpha channel.") + LOGGER.warning("Discarding alpha channel.") return RGBImage(image) diff --git a/cellprofiler_core/image/abstract_image/_abstract_image.py b/cellprofiler_core/image/abstract_image/_abstract_image.py index 0f600f36..b2b94fb6 100644 --- a/cellprofiler_core/image/abstract_image/_abstract_image.py +++ b/cellprofiler_core/image/abstract_image/_abstract_image.py @@ -1,6 +1,8 @@ import logging +LOGGER = logging.getLogger(__name__) + class AbstractImage: """Represents an image provider that returns images """ @@ -22,7 +24,7 @@ def get_name(self): def release_memory(self): """Release whatever memory is associated with the image""" - logging.warning( + LOGGER.warning( "Warning: no memory release function implemented for %s image", self.get_name(), ) diff --git a/cellprofiler_core/image/abstract_image/file/_file_image.py b/cellprofiler_core/image/abstract_image/file/_file_image.py index 0b89c4ef..6482bd43 100644 --- a/cellprofiler_core/image/abstract_image/file/_file_image.py +++ b/cellprofiler_core/image/abstract_image/file/_file_image.py @@ -278,7 +278,7 @@ def release_memory(self): try: os.remove(self.__cached_file) except: - logging.warning( + LOGGER.warning( "Could not delete file %s", self.__cached_file, exc_info=True ) self.__is_cached = False diff --git a/cellprofiler_core/measurement/_measurements.py b/cellprofiler_core/measurement/_measurements.py index 70effc00..8570e624 100644 --- a/cellprofiler_core/measurement/_measurements.py +++ b/cellprofiler_core/measurement/_measurements.py @@ -62,6 +62,8 @@ from ..utilities.measurement import make_temporary_file +LOGGER = logging.getLogger(__name__) + class Measurements: """Represents measurements made on images and objects """ @@ -108,9 +110,9 @@ def __init__( is_temporary = True import traceback - logging.debug("Created temporary file %s" % filename) + LOGGER.debug("Created temporary file %s" % filename) for frame in traceback.extract_stack(): - logging.debug("{}: ({} {}): {}".format(*frame)) + LOGGER.debug("{}: ({} {}): {}".format(*frame)) else: is_temporary = False @@ -1273,7 +1275,7 @@ def load_image_sets(self, fd_or_file, start=None, stop=None): column.append(field) last_image_number = image_number if last_image_number == 0: - logging.warn("No image sets were loaded") + LOGGER.warn("No image sets were loaded") return if start is None: image_numbers = list(range(1, last_image_number + 1)) @@ -1606,7 +1608,7 @@ def get_image( ) if image.pixel_data.shape[-1] == 4: - logging.warning("Discarding alpha channel.") + LOGGER.warning("Discarding alpha channel.") return RGBImage(image) diff --git a/cellprofiler_core/module/_module.py b/cellprofiler_core/module/_module.py index 462bd298..827c4726 100644 --- a/cellprofiler_core/module/_module.py +++ b/cellprofiler_core/module/_module.py @@ -21,6 +21,8 @@ from ..setting import ValidationError +LOGGER = logging.getLogger(__name__) + class Module: """ Derive from the abstract module class to create your own module in Python @@ -111,7 +113,7 @@ def from_dict(self, settings: list, attributes: dict): self.module_name = attributes["module_name"] setting_values = [setting["value"] for setting in settings] if attributes["variable_revision_number"] > self.variable_revision_number: - logging.warning(f"Loaded module '{self.module_name}' is from a newer version of CellProfiler - " + LOGGER.warning(f"Loaded module '{self.module_name}' is from a newer version of CellProfiler - " f"v{attributes['variable_revision_number']}, current version is " f"v{self.variable_revision_number}. Settings may load incorrectly.") self.set_settings_from_values( diff --git a/cellprofiler_core/modules/images.py b/cellprofiler_core/modules/images.py index 24304d21..18b8e5ff 100644 --- a/cellprofiler_core/modules/images.py +++ b/cellprofiler_core/modules/images.py @@ -24,6 +24,9 @@ from cellprofiler_core.utilities.image import image_resource from cellprofiler_core.utilities.image import is_image + +LOGGER = logging.getLogger(__name__) + __doc__ = """\ Images ====== @@ -290,7 +293,7 @@ def on_activated(self, workspace): def filter_file_list(self, workspace): file_list = workspace.pipeline.file_list if file_list and not workspace.file_list.has_files(): - logging.warning("Workspace file list is empty, will populate from pipeline." + LOGGER.warning("Workspace file list is empty, will populate from pipeline." "This may happen if you're running in headless mode.") workspace.file_list.add_files_to_filelist([f.url for f in file_list]) if self.filter_choice != FILTER_CHOICE_NONE: @@ -314,7 +317,7 @@ def prepare_run(self, workspace): return True file_list = self.filter_file_list(workspace) if self.want_split.value: - logging.debug("Metadata extraction will be performed now if needed") + LOGGER.debug("Metadata extraction will be performed now if needed") if self.extract_metadata.callback is not None: # If GUI is present perform extraction and refresh the file list GUI self.extract_metadata.callback() diff --git a/cellprofiler_core/modules/metadata.py b/cellprofiler_core/modules/metadata.py index e5ea73bc..976ac173 100644 --- a/cellprofiler_core/modules/metadata.py +++ b/cellprofiler_core/modules/metadata.py @@ -73,6 +73,9 @@ from cellprofiler_core.utilities.image import image_resource from cellprofiler_core.utilities.measurement import find_metadata_tokens + +LOGGER = logging.getLogger(__name__) + __doc__ = """\ Metadata ======== @@ -1130,13 +1133,13 @@ def apply_data_type(value, data_type): try: return int(value) except ValueError: - logging.warning(f"Metadata value {value} cannot be interpreted as an integer number.") + LOGGER.warning(f"Metadata value {value} cannot be interpreted as an integer number.") return value elif data_type == DataTypes.DT_FLOAT: try: return float(value) except ValueError: - logging.warning(f"Metadata value {value} cannot be interpreted as a floating point number.") + LOGGER.warning(f"Metadata value {value} cannot be interpreted as a floating point number.") return value return value diff --git a/cellprofiler_core/modules/namesandtypes.py b/cellprofiler_core/modules/namesandtypes.py index aa0078df..01891218 100644 --- a/cellprofiler_core/modules/namesandtypes.py +++ b/cellprofiler_core/modules/namesandtypes.py @@ -89,6 +89,9 @@ ) from ..utilities.image import image_resource + +LOGGER = logging.getLogger(__name__) + __doc__ = """\ NamesAndTypes ============= @@ -1255,7 +1258,7 @@ def make_image_sets_by_order(self, image_planes): break errors = [] if not groups: - logging.warning("No images passed group filters") + LOGGER.warning("No images passed group filters") return [] desired_length = max([len(grp) for grp in groups.values()]) for name in self.get_column_names(want_singles=False): @@ -1375,7 +1378,7 @@ def handle_error_messages(errors): text = f"Metadata {error_info} for channel {error_chan} had {error_type}" else: text = f"Channel {error_chan} had {error_type}" - logging.warning(text) + LOGGER.warning(text) error_types[(error_type, error_chan)] += 1 if not get_headless(): msg = f"Warning: found {len(errors)} image set errors (see log for details)\n \n" diff --git a/cellprofiler_core/pipeline/_image_file.py b/cellprofiler_core/pipeline/_image_file.py index d8656c12..e533d926 100644 --- a/cellprofiler_core/pipeline/_image_file.py +++ b/cellprofiler_core/pipeline/_image_file.py @@ -13,7 +13,7 @@ from cellprofiler_core.reader import get_image_reader, Reader from cellprofiler_core.utilities.image import url_to_modpath, is_file_url -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class ImageFile: @@ -85,12 +85,12 @@ def extract_planes(self, workspace=None): try: reader = self.get_reader() except: - logger.error(f"May not be an image: {self.url}", exc_info=True) + LOGGER.error(f"May not be an image: {self.url}", exc_info=True) self.metadata[MD_SIZE_S] = 0 return meta_dict = reader.get_series_metadata() if meta_dict[MD_SIZE_S] == 0: - logger.error(f"File {self.filename} appears to contain no images.") + LOGGER.error(f"File {self.filename} appears to contain no images.") self.metadata[MD_SIZE_S] = 0 self.release_reader() return @@ -118,7 +118,7 @@ def load_plane_metadata(self, data, names=''): if len(data) < 5 or len(data) % 5 != 0: # No metadata or bad format if len(data) > 0: - logger.warning(f"Unable to load saved metadata for {self.filename}") + LOGGER.warning(f"Unable to load saved metadata for {self.filename}") return if len(data) == 5 and numpy.all(data == -1): # Unfilled metadata @@ -221,7 +221,7 @@ def add_metadata(self, meta_dict): # Used to bulk-add metadata keys from the Metadata module. Other modules should use set_metadata method. for key, value in meta_dict.items(): if key in RESERVED_METADATA_KEYS: - logger.error(f"Unable to set protected metadata key '{key}' to '{value}' for file {self.filename}. " + LOGGER.error(f"Unable to set protected metadata key '{key}' to '{value}' for file {self.filename}. " f"Please choose another key name.") continue self._metadata_dict[key] = value @@ -231,7 +231,7 @@ def set_metadata(self, key, value, force=False): raise PermissionError(f"Cannot override protected metadata key '{key}'") else: if force: - logger.warning(f"Overwriting protected key {key}. This may break functionality.") + LOGGER.warning(f"Overwriting protected key {key}. This may break functionality.") self._metadata_dict[key] = value def clear_metadata(self): diff --git a/cellprofiler_core/pipeline/_image_plane.py b/cellprofiler_core/pipeline/_image_plane.py index b17d7e5a..0b8825a0 100644 --- a/cellprofiler_core/pipeline/_image_plane.py +++ b/cellprofiler_core/pipeline/_image_plane.py @@ -6,7 +6,7 @@ C_SERIES, C_CHANNEL, C_Z, C_T, C_INDEX, C_SERIES_NAME from cellprofiler_core.pipeline import ImageFile -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class ImagePlane: @@ -166,7 +166,7 @@ def set_metadata(self, key, value, force=False): raise PermissionError(f"Cannot override protected metadata key '{key}'") else: if force: - logger.warning(f"Overwriting protected key {key}. This may break functionality.") + LOGGER.warning(f"Overwriting protected key {key}. This may break functionality.") self._metadata_dict[key] = value diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index 4363ac04..de7c767b 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -99,6 +99,9 @@ from cellprofiler_core import __version__ +LOGGER = logging.getLogger(__name__) + + def _is_fp(x): return hasattr(x, "seek") and hasattr(x, "read") @@ -234,7 +237,7 @@ def reload_modules(self): self.loadtxt(fd, raise_on_error=True) return True except Exception as e: - logging.warning( + LOGGER.warning( "Modules reloaded, but could not reinstantiate pipeline with new versions.", exc_info=True, ) @@ -258,7 +261,7 @@ def is_pipeline_txt_fd(fd): if header.startswith("CellProfiler Pipeline: http://www.cellprofiler.org"): return True if re.search(SAD_PROOFPOINT_COOKIE, header): - logging.info('print_emoji(":cat_crying_because_of_proofpoint:")') + LOGGER.info('print_emoji(":cat_crying_because_of_proofpoint:")') return True return False @@ -347,7 +350,7 @@ def load(self, fd_or_filename): @staticmethod def respond_to_version_mismatch_error(message): - logging.warning(message) + LOGGER.warning(message) def loadtxt(self, fp_or_filename, raise_on_error=False): """Load a pipeline from a text file @@ -485,7 +488,7 @@ def validate_pipeline_version(self, current_version, git_hash, pipeline_version) "this pipeline or if you will only use it with this or\n" "later versions of CellProfiler." ).format(git_hash, pipeline_date) - logging.warning(message) + LOGGER.warning(message) else: # pipeline versions pre-3.0.0 have unpredictable formatting if pipeline_version == 300: @@ -506,7 +509,7 @@ def validate_pipeline_version(self, current_version, git_hash, pipeline_version) "this pipeline or if you will only use it with this or\n" "later versions of CellProfiler." ).format(pipeline_version) - logging.warning(message) + LOGGER.warning(message) return pipeline_version @@ -571,7 +574,7 @@ def setup_modules(self, fd, module_count, raise_on_error): except Exception as instance: if raise_on_error: raise - logging.error("Failed to load pipeline", exc_info=True) + LOGGER.error("Failed to load pipeline", exc_info=True) event = LoadException(instance, module, module_name, settings) self.notify_listeners(event) if event.cancel_run: @@ -924,7 +927,7 @@ def run_with_yield( last_image_number = None - logging.info("Times reported are CPU and Wall-clock times for each module") + LOGGER.info("Times reported are CPU and Wall-clock times for each module") __group = self.group( grouping, @@ -1032,7 +1035,7 @@ def run_with_yield( self.run_module(module, workspace) except Exception as instance: print("pipeline_exception") - logging.error( + LOGGER.error( "Error detected during run of module %s", module.module_name, exc_info=True, @@ -1051,7 +1054,7 @@ def run_with_yield( cpu_delta_sec = max(0, cpu_t1 - cpu_t0) wall_delta_sec = max(0, wall_t1 - wall_t0) - logging.info( + LOGGER.info( "%s: Image # %d, module %s # %d: CPU_time = %.2f secs, Wall_time = %.2f secs" % ( start_time.ctime(), @@ -1071,7 +1074,7 @@ def run_with_yield( fig.Refresh() except Exception as instance: - logging.error( + LOGGER.error( "Failed to display results for module %s", module.module_name, exc_info=True, @@ -1230,18 +1233,18 @@ def run_image_set( if module in self.redundancy_map and len(self.modules()) > module.module_num: to_forget = self.redundancy_map[module] for image_name in to_forget: - logging.info(f"Releasing memory for redundant image {image_name}") + LOGGER.info(f"Releasing memory for redundant image {image_name}") workspace.image_set.clear_image(image_name) gc.collect() except Exception as e: - logging.warning(f"Encountered error during memory cleanup: {e}") + LOGGER.warning(f"Encountered error during memory cleanup: {e}") except CancelledException: # Analysis worker interaction handler is telling us that # the UI has cancelled the run. Forward exception upward. raise except Exception as exception: print("run_image_set_exception, get_always_continue",get_always_continue()) - logging.error( + LOGGER.error( "Error detected during run of module %s#%d", module.module_name, module.module_num, @@ -1265,7 +1268,7 @@ def run_image_set( cpu_t1 = sum(os_times[:-1]) cpu_delta_secs = max(0, cpu_t1 - cpu_t0) wall_delta_secs = max(0, wall_t1 - wall_t0) - logging.info( + LOGGER.info( "%s: Image # %d, module %s # %d: CPU_time = %.2f secs, Wall_time = %.2f secs" % ( start_time.ctime(), @@ -1493,7 +1496,7 @@ def on_pipeline_event(pipeline, event): self.clear_measurements(workspace.measurements) break except Exception as instance: - logging.error( + LOGGER.error( "Failed to prepare run for module %s", module.module_name, exc_info=True, @@ -1596,7 +1599,7 @@ def post_run(self, *args): try: module.post_run(workspace) except Exception as instance: - logging.error( + LOGGER.error( "Failed to complete post_run processing for module %s." % module.module_name, exc_info=True, @@ -1613,7 +1616,7 @@ def post_run(self, *args): workspace.post_run_display(module) except Exception as instance: # Warn about display failure but keep going. - logging.warning( + LOGGER.warning( "Caught exception during post_run_display for module %s." % module.module_name, exc_info=True, @@ -1645,7 +1648,7 @@ def prepare_to_create_batch(self, workspace, fn_alter_path): workspace.set_module(module) module.prepare_to_create_batch(workspace, fn_alter_path) except Exception as instance: - logging.error( + LOGGER.error( "Failed to collect batch information for module %s", module.module_name, exc_info=True, @@ -1741,7 +1744,7 @@ def prepare_group(self, workspace, grouping, image_numbers): try: module.prepare_group(workspace, grouping, image_numbers) except Exception as instance: - logging.error( + LOGGER.error( "Failed to prepare group in module %s", module.module_name, exc_info=True, @@ -1763,7 +1766,7 @@ def post_group(self, workspace, grouping): try: module.post_group(workspace, grouping) except Exception as instance: - logging.error( + LOGGER.error( "Failed during post-group processing for module %s" % module.module_name, exc_info=True, @@ -1779,7 +1782,7 @@ def post_group(self, workspace, grouping): try: workspace.post_group_display(module) except: - logging.warning( + LOGGER.warning( "Failed during post group display for module %s" % module.module_name, exc_info=True, @@ -1891,7 +1894,7 @@ def undo(): def enable_module(self, module): """Enable a module = make it executable""" if module.enabled: - logging.warning( + LOGGER.warning( "Asked to enable module %s, but it was already enabled" % module.module_name ) @@ -1908,7 +1911,7 @@ def undo(): def disable_module(self, module): """Disable a module = prevent it from being executed""" if not module.enabled: - logging.warning( + LOGGER.warning( "Asked to disable module %s, but it was already disabled" % module.module_name ) @@ -2036,7 +2039,7 @@ def load_file_list(self, workspace): try: urls, metadata, series_names = file_list.get_filelist(want_metadata=True) except Exception as instance: - logging.error("Failed to get file list from workspace", exc_info=True) + LOGGER.error("Failed to get file list from workspace", exc_info=True) x = IPDLoadException("Failed to get file list from workspace") self.notify_listeners(x) if x.cancel_run: diff --git a/cellprofiler_core/pipeline/io/_v6.py b/cellprofiler_core/pipeline/io/_v6.py index 70f0d2bd..8fc7775c 100644 --- a/cellprofiler_core/pipeline/io/_v6.py +++ b/cellprofiler_core/pipeline/io/_v6.py @@ -3,7 +3,7 @@ import os import re -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) import cellprofiler_core from cellprofiler_core.constants.pipeline import ( @@ -55,7 +55,7 @@ def load(pipeline, fd): pipeline_dict = json.load(fd) cp_version = int(re.sub(r"\.|rc\d", "", cellprofiler_core.__version__)) if cp_version != pipeline_dict['date_revision']: - logging.warning(f"Pipeline file is from a different version of CellProfiler. " + LOGGER.warning(f"Pipeline file is from a different version of CellProfiler. " f"Current:v{cp_version} File:v{pipeline_dict['date_revision']}." f" Will attempt to upgrade settings.") pipeline_modules = pipeline.modules(False) diff --git a/cellprofiler_core/preferences/__init__.py b/cellprofiler_core/preferences/__init__.py index e465807a..4bc12382 100644 --- a/cellprofiler_core/preferences/__init__.py +++ b/cellprofiler_core/preferences/__init__.py @@ -25,6 +25,9 @@ from ..constants.reader import ALL_READERS from ..utilities.image import image_resource + +LOGGER = logging.getLogger(__name__) + """get_absolute_path - mode = output. Assume "." is the default output dir""" ABSPATH_OUTPUT = "abspath_output" @@ -113,12 +116,12 @@ def get_config(): try: preferences_version_number = int(config_read(PREFERENCES_VERSION)) if preferences_version_number != PREFERENCES_VERSION_NUMBER: - logging.warning( + LOGGER.warning( "Preferences version mismatch: expected %d, at %d" % (PREFERENCES_VERSION_NUMBER, preferences_version_number) ) except: - logging.warning( + LOGGER.warning( "Preferences version was %s, not a number. Resetting to current version" % preferences_version_number ) @@ -302,7 +305,7 @@ def config_write_typed(key, value, key_type=None, flush=True): else: success = config.Write(key, value) if not success: - logging.error(f"Unable to write preference key {key}") + LOGGER.error(f"Unable to write preference key {key}") return if flush: config.Flush() @@ -319,7 +322,7 @@ def export_to_json(path): del config_dict[key] with open(path, mode='wt') as fd: json.dump(config_dict, fd) - logging.info(f"Wrote config to {path}") + LOGGER.info(f"Wrote config to {path}") def cell_profiler_root_directory(): @@ -842,10 +845,10 @@ def get_default_image_directory(): __default_image_directory = os.path.normcase(default_image_directory) return __default_image_directory except: - logging.error( + LOGGER.error( "Unknown failure when retrieving the default image directory", exc_info=True ) - logging.warning( + LOGGER.warning( "Warning: current path of %s is not a valid directory. Switching to home directory." % (default_image_directory.encode("ascii", "replace")) ) @@ -912,11 +915,11 @@ def get_default_output_directory(): __default_output_directory = os.path.normcase(default_output_directory) return __default_output_directory except: - logging.error( + LOGGER.error( "Unknown failure when retrieving the default output directory", exc_info=True, ) - logging.warning( + LOGGER.warning( "Warning: current path of %s is not a valid directory. Switching to home directory." % (default_output_directory.encode("ascii", "replace")) ) diff --git a/cellprofiler_core/readers/ngff_reader.py b/cellprofiler_core/readers/ngff_reader.py index a04b0850..09d2580d 100644 --- a/cellprofiler_core/readers/ngff_reader.py +++ b/cellprofiler_core/readers/ngff_reader.py @@ -62,7 +62,7 @@ def get_reader(self): # Zarr has consolidated metadata. self._reader = zarr.convenience.open_consolidated(store, mode='r') else: - logging.warning(f"Image is on S3 but lacks consolidated metadata. " + LOGGER.warning(f"Image is on S3 but lacks consolidated metadata. " f"This may degrade reading performance. URL: {self.root}") self._reader = zarr.open(store, mode='r') elif not os.path.isdir(store.path): @@ -71,7 +71,7 @@ def get_reader(self): self._reader = zarr.open(store, mode='r') NGFFReader.ZARR_READER_CACHE[self.root] = self._reader, None if self.group: - logging.warning(f"Reader had a group? {self.group}") + LOGGER.warning(f"Reader had a group? {self.group}") return self._reader[self.group] return self._reader @@ -104,7 +104,7 @@ def read(self, return a tuple of image and max intensity :param channel_names: provide the channel names for the OME metadata """ - logging.debug(f"Reading {c=}, {z=}, {t=}, {series=}, {index=}, {xywh=}") + LOGGER.debug(f"Reading {c=}, {z=}, {t=}, {series=}, {index=}, {xywh=}") c2 = None if c is None else c + 1 z2 = None if z is None else z + 1 t2 = None if t is None else t + 1 diff --git a/cellprofiler_core/setting/_image_plane.py b/cellprofiler_core/setting/_image_plane.py index a28e0e02..251bc6ae 100644 --- a/cellprofiler_core/setting/_image_plane.py +++ b/cellprofiler_core/setting/_image_plane.py @@ -5,6 +5,8 @@ from ._validation_error import ValidationError +LOGGER = logging.getLogger(__name__) + class ImagePlane(Setting): """A setting that specifies an image plane @@ -55,7 +57,7 @@ def build(plane): if " " in url: # Spaces are not legal characters in URLs, nevertheless, I try # to accommodate - logging.warning( + LOGGER.warning( "URLs should not contain spaces. %s is the offending URL" % url ) url = url.replace(" ", "%20") diff --git a/cellprofiler_core/setting/range/_range.py b/cellprofiler_core/setting/range/_range.py index df558e48..e573252a 100644 --- a/cellprofiler_core/setting/range/_range.py +++ b/cellprofiler_core/setting/range/_range.py @@ -4,6 +4,8 @@ from .._validation_error import ValidationError +LOGGER = logging.getLogger(__name__) + class Range(Setting): """A setting representing a range between two values""" @@ -112,7 +114,7 @@ def set_value_text(self, value): self.__default_min = self.min self.__default_max = self.max except: - logging.debug("Illegal value in range setting: %s" % value) + LOGGER.debug("Illegal value in range setting: %s" % value) def test_valid(self, pipeline): values = self.value_text.split(",") diff --git a/cellprofiler_core/setting/text/number/_number.py b/cellprofiler_core/setting/text/number/_number.py index 50a68346..a93f728a 100644 --- a/cellprofiler_core/setting/text/number/_number.py +++ b/cellprofiler_core/setting/text/number/_number.py @@ -4,6 +4,8 @@ from ..._validation_error import ValidationError +LOGGER = logging.getLogger(__name__) + class Number(Text): """A setting that allows only numeric input """ @@ -55,7 +57,7 @@ def set_value_text(self, value_text): self.test_valid(None) self.__default = self.str_to_value(value_text) except: - logging.debug("Number set to illegal value: %s" % value_text) + LOGGER.debug("Number set to illegal value: %s" % value_text) def set_min_value(self, minval): """Programatically set the minimum value allowed""" diff --git a/cellprofiler_core/utilities/core/modules/__init__.py b/cellprofiler_core/utilities/core/modules/__init__.py index 40cfc6cf..13e7bec7 100644 --- a/cellprofiler_core/utilities/core/modules/__init__.py +++ b/cellprofiler_core/utilities/core/modules/__init__.py @@ -20,6 +20,8 @@ from ..plugins import load_plugins +LOGGER = logging.getLogger(__name__) + def check_module(module, name): if hasattr(module, "do_not_check"): return @@ -68,14 +70,14 @@ def add_module(mod, check_svn, class_name=None): cp_module = find_cpmodule(m) name = cp_module.module_name except Exception as e: - logging.warning("Could not load %s", mod, exc_info=True) + LOGGER.warning("Could not load %s", mod, exc_info=True) badmodules.append((mod, e)) return try: pymodules.append(m) if name in all_modules: - logging.warning( + LOGGER.warning( "Multiple definitions of module %s\n\told in %s\n\tnew in %s", name, sys.modules[all_modules[name].__module__].__file__, @@ -91,7 +93,7 @@ def add_module(mod, check_svn, class_name=None): if match is not None: svn_revisions[name] = match.groups()[0] except Exception as e: - logging.warning("Failed to load %s", name, exc_info=True) + LOGGER.warning("Failed to load %s", name, exc_info=True) badmodules.append((mod, e)) if name in all_modules: del all_modules[name] @@ -114,7 +116,7 @@ def add_module(mod, check_svn, class_name=None): add_module("cellprofiler.modules." + modname, True, class_name=classname) if len(badmodules) > 0: - logging.warning( + LOGGER.warning( "could not load these modules: %s", ",".join([x[0] for x in badmodules]) ) diff --git a/cellprofiler_core/utilities/hdf5_dict.py b/cellprofiler_core/utilities/hdf5_dict.py index a682aaa8..6ec372bb 100644 --- a/cellprofiler_core/utilities/hdf5_dict.py +++ b/cellprofiler_core/utilities/hdf5_dict.py @@ -22,10 +22,12 @@ import cellprofiler_core.utilities.legacy -install_aliases() LOGGER = logging.getLogger(__name__) +install_aliases() + + version_number = 1 VERSION = "Version" diff --git a/cellprofiler_core/utilities/zmq/__init__.py b/cellprofiler_core/utilities/zmq/__init__.py index 85f2852c..173d0b91 100644 --- a/cellprofiler_core/utilities/zmq/__init__.py +++ b/cellprofiler_core/utilities/zmq/__init__.py @@ -18,6 +18,9 @@ Request, ) + +LOGGER = logging.getLogger(__name__) + NOTIFY_SOCKET_ADDR = "inproc://BoundaryNotifications" SD_KEY_DICT = "__keydict__" @@ -256,10 +259,10 @@ def lock_thread_fn(): request = msg[2] assert isinstance(request, LockStatusRequest) assert isinstance(boundary, Boundary) - logging.info("Received lock status request for %s" % request.uid) + LOGGER.info("Received lock status request for %s" % request.uid) reply = LockStatusReply(request.uid, request.uid in locked_uids) if reply.locked: - logging.info( + LOGGER.info( "Denied lock request for %s" % locked_uids[request.uid] ) boundary.enqueue_reply(request, reply) @@ -277,7 +280,7 @@ def lock_thread_fn(): except Exception as e: msg[3].put(e) __lock_thread = None - logging.info("Exiting the lock thread") + LOGGER.info("Exiting the lock thread") __lock_thread = threading.Thread(target=lock_thread_fn) __lock_thread.setName("FileLockThread") @@ -309,12 +312,12 @@ def lock_file(path, timeout=3): except OSError as e: if e.errno != errno.EEXIST: raise - logging.info("Lockfile for %s already exists - contacting owner" % path) + LOGGER.info("Lockfile for %s already exists - contacting owner" % path) with open(lock_path, "r") as f: remote_address = f.readline().strip() remote_uid = f.readline().strip() if len(remote_address) > 0 and len(remote_uid) > 0: - logging.info("Owner is %s" % remote_address) + LOGGER.info("Owner is %s" % remote_address) request_socket = the_boundary.zmq_context.socket(zmq.REQ) request_socket.setsockopt(zmq.LINGER, 0) assert isinstance(request_socket, zmq.Socket) @@ -333,7 +336,7 @@ def lock_file(path, timeout=3): lock_response = lock_request.recv(socket) if isinstance(lock_response, LockStatusReply): if lock_response.locked: - logging.info("%s is locked" % path) + LOGGER.info("%s is locked" % path) return False keep_polling = False # diff --git a/cellprofiler_core/utilities/zmq/_boundary.py b/cellprofiler_core/utilities/zmq/_boundary.py index dea4c0bc..c7d5ef9b 100644 --- a/cellprofiler_core/utilities/zmq/_boundary.py +++ b/cellprofiler_core/utilities/zmq/_boundary.py @@ -12,6 +12,8 @@ from ...constants.worker import NOTIFY_STOP, NOTIFY_RUN +LOGGER = logging.getLogger(__name__) + class Boundary: """This object serves as the interface between a ZMQ socket passing Requests and Replies, and a thread or threads serving those requests. @@ -89,7 +91,7 @@ def register_analysis(self, analysis_id, upward_queue): self.analysis_context = AnalysisContext( analysis_id, upward_queue, self.analysis_context_lock ) - logging.debug(f"Registered analysis as id {analysis_id}") + LOGGER.debug(f"Registered analysis as id {analysis_id}") def register_request_class(self, cls_request, upward_queue): """Register a queue to receive requests of the given class @@ -191,7 +193,7 @@ def spin(self): msg = selfnotify_socket.recv() # Let's watch out for stop signals anyway if msg == NOTIFY_STOP: - logging.warning("Captured a stop message over zmq") + LOGGER.warning("Captured a stop message over zmq") received_stop = True continue req = Communicable.recv(s, routed=True) @@ -203,7 +205,7 @@ def spin(self): q.put([self, self.NOTIFY_REQUEST, req]) break else: - logging.warning( + LOGGER.warning( "Received a request that wasn't an AnalysisRequest: %s" % str(type(req)) ) @@ -211,7 +213,7 @@ def spin(self): continue if s != request_socket: # Request is on the external socket - logging.warning("Received a request on the external socket") + LOGGER.warning("Received a request on the external socket") req.reply(BoundaryExited()) continue # @@ -251,13 +253,13 @@ def spin(self): request_socket.close() selfnotify_socket.close() - logging.debug("Shutting down the boundary thread") + LOGGER.debug("Shutting down the boundary thread") except: # # Pretty bad - a logic error or something extremely unexpected # We're close to hosed here, better die an ugly death. # - logging.critical("Unhandled exception in boundary thread.", + LOGGER.critical("Unhandled exception in boundary thread.", exc_info=True) import os os._exit(-1) diff --git a/cellprofiler_core/worker/__init__.py b/cellprofiler_core/worker/__init__.py index ed2688a1..a587c8d2 100644 --- a/cellprofiler_core/worker/__init__.py +++ b/cellprofiler_core/worker/__init__.py @@ -34,6 +34,9 @@ from cellprofiler_core.preferences import set_always_continue, set_conserve_memory from cellprofiler_core.worker._worker import Worker + +LOGGER = logging.getLogger(__name__) + """Set the log level through the environment by specifying AW_LOG_LEVEL""" AW_LOG_LEVEL = "AW_LOG_LEVEL" @@ -144,7 +147,7 @@ def aw_parse_args(): if options.always_continue is not None: set_always_continue(options.always_continue, globally=False) else: - logging.warning("Plugins directory not set") + LOGGER.warning("Plugins directory not set") if __name__ == "__main__": @@ -249,14 +252,14 @@ def main(): ): worker_thread.join() the_zmq_context.destroy(linger=0) - logging.debug("Worker thread joined") + LOGGER.debug("Worker thread joined") # # Shutdown - need to handle some global cleanup here # try: stop_java() except: - logging.warning("Failed to stop the JVM", exc_info=True) + LOGGER.warning("Failed to stop the JVM", exc_info=True) def monitor_keepalive(context, keepalive_address): @@ -289,7 +292,7 @@ def monitor_keepalive(context, keepalive_address): event = keepalive_socket.poll(5000) if not event: missed += 1 - logging.warning(f"Worker failed to receive communication for" + LOGGER.warning(f"Worker failed to receive communication for" f" {5 * missed} seconds") else: missed = 0 @@ -301,12 +304,12 @@ def monitor_keepalive(context, keepalive_address): keepalive_socket.close() if missed >= 3: # Parent is dead, hard kill - logging.critical("Worker stopped receiving communication from " + LOGGER.critical("Worker stopped receiving communication from " "CellProfiler, shutting down now") else: # Stop signal captured, give some time to shut down gracefully. time.sleep(10) - logging.info("Cancelling worker") + LOGGER.info("Cancelling worker") # hard exit after 10 seconds unless app exits for m in all_measurements: @@ -314,7 +317,7 @@ def monitor_keepalive(context, keepalive_address): m.close() except: pass - logging.error("Worker failed to stop gracefully, forcing exit now") + LOGGER.error("Worker failed to stop gracefully, forcing exit now") os._exit(0) diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index b26bb858..3eb0995a 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -32,6 +32,8 @@ from ..workspace import Workspace +LOGGER = logging.getLogger(__name__) + class Worker: """An analysis worker processing work at a given address @@ -106,7 +108,7 @@ def run(self): with self.AnalysisWorkerThreadObject(self): while not self.cancelled: try: - logging.debug("Requesting a job") + LOGGER.debug("Requesting a job") # fetch a job the_request = Work(self.current_analysis_id) job = self.send(the_request) @@ -130,26 +132,26 @@ def do_job(self, job): try: send_dictionary = job.wants_dictionary - logging.info("Starting job") + LOGGER.info("Starting job") # Fetch the pipeline and preferences for this analysis if we don't have it current_pipeline = self.pipeline current_preferences = self.preferences if not current_pipeline: - logging.debug("Fetching pipeline and preferences") + LOGGER.debug("Fetching pipeline and preferences") rep = self.send(PipelinePreferences(self.current_analysis_id)) - logging.debug("Received pipeline and preferences response") + LOGGER.debug("Received pipeline and preferences response") preferences_dict = rep.preferences # update preferences to match remote values set_preferences_from_dict(preferences_dict) - logging.debug("Loading pipeline") + LOGGER.debug("Loading pipeline") current_pipeline = cpp.Pipeline() pipeline_chunks = rep.pipeline_blob.tolist() pipeline_io = io.StringIO("".join(pipeline_chunks)) current_pipeline.loadtxt(pipeline_io, raise_on_error=True) - logging.debug("Pipeline loaded") + LOGGER.debug("Pipeline loaded") current_pipeline.add_listener(self.pipeline_listener.handle_event) current_preferences = rep.preferences self.pipeline = current_pipeline @@ -161,16 +163,16 @@ def do_job(self, job): # Reset the listener's state self.pipeline_listener.reset() - logging.debug("Getting initial measurements") + LOGGER.debug("Getting initial measurements") # Fetch the path to the intial measurements if needed. if self.initial_measurements is None: - logging.debug("Sending initial measurements request") + LOGGER.debug("Sending initial measurements request") rep = self.send(InitialMeasurements(self.current_analysis_id)) - logging.debug("Got initial measurements") + LOGGER.debug("Got initial measurements") self.initial_measurements = load_measurements_from_buffer(rep.buf) else: - logging.debug("Has initial measurements") + LOGGER.debug("Has initial measurements") # Make a copy of the measurements for writing during this job current_measurements = Measurements(copy=self.initial_measurements) all_measurements.add(current_measurements) @@ -179,7 +181,7 @@ def do_job(self, job): successful_image_set_numbers = [] image_set_numbers = job.image_set_numbers worker_runs_post_group = job.worker_runs_post_group - logging.info("Doing job: " + ",".join(map(str, image_set_numbers))) + LOGGER.info("Doing job: " + ",".join(map(str, image_set_numbers))) self.pipeline_listener.image_set_number = image_set_numbers[0] @@ -253,11 +255,11 @@ def do_job(self, job): ) rep = self.send(req) except CancelledException: - logging.info("Aborting job after cancellation") + LOGGER.info("Aborting job after cancellation") abort = True except Exception as e: try: - logging.error("Error in pipeline", exc_info=True) + LOGGER.error("Error in pipeline", exc_info=True) if ( self.handle_exception(image_set_number=image_set_number) == ED_STOP @@ -265,7 +267,7 @@ def do_job(self, job): abort = True break except: - logging.error( + LOGGER.error( "Error in handling of pipeline exception", exc_info=True ) # this is bad. We can't handle nested exceptions @@ -304,7 +306,7 @@ def do_job(self, job): raise except Exception: - logging.error("Error in worker", exc_info=True) + LOGGER.error("Error in worker", exc_info=True) if self.handle_exception() == ED_STOP: raise CancelledException("Cancelling after user-requested stop") finally: @@ -390,14 +392,14 @@ def send(self, req, work_socket=None): elif socket == self.keepalive_socket: notify_msg = self.keepalive_socket.recv() if notify_msg == NOTIFY_STOP: - logging.debug("Worker received cancel notification") + LOGGER.debug("Worker received cancel notification") self.cancelled = True self.raise_cancel( "Received stop notification while waiting for " "response from %s" % str(req) ) else: - logging.error("Unexpected message on keepalive: " + notify_msg.decode()) + LOGGER.error("Unexpected message on keepalive: " + notify_msg.decode()) elif socket == work_socket: response = req.recv(work_socket) if isinstance(response, (UpstreamExit, ServerExited)): @@ -418,7 +420,7 @@ def raise_cancel(self, msg="Cancelling analysis"): """ from cellprofiler_core.pipeline.event import CancelledException - logging.debug(msg) + LOGGER.debug(msg) self.cancelled = True if self.initial_measurements is not None: self.initial_measurements.close() diff --git a/cellprofiler_core/workspace/_workspace.py b/cellprofiler_core/workspace/_workspace.py index 116de774..2a96ff6c 100644 --- a/cellprofiler_core/workspace/_workspace.py +++ b/cellprofiler_core/workspace/_workspace.py @@ -14,6 +14,8 @@ from ..utilities.hdf5_dict import HDF5FileList +LOGGER = logging.getLogger(__name__) + class Workspace: """The workspace contains the processing information and state for a pipeline run on an image set @@ -536,7 +538,7 @@ def refresh_image_set(self, force=False): result = self.pipeline.prepare_run(self, stop_module) return result except: - logging.error("Failed during prepare_run", exc_info=1) + LOGGER.error("Failed during prepare_run", exc_info=1) return False finally: if no_image_set_list: @@ -561,7 +563,7 @@ def notify(self, event): try: callback(event) except: - logging.error("Notification callback threw an exception", exc_info=1) + LOGGER.error("Notification callback threw an exception", exc_info=1) def __on_file_list_changed(self): self.notify(self.WorkspaceFileListNotification(self)) diff --git a/tests/analysis/test_analysis.py b/tests/analysis/test_analysis.py index 551b37ca..91ce1092 100644 --- a/tests/analysis/test_analysis.py +++ b/tests/analysis/test_analysis.py @@ -3,31 +3,25 @@ import logging import queue - import pytest -from importlib.util import find_spec - -import cellprofiler_core.constants.measurement -import cellprofiler_core.utilities.measurement -from cellprofiler_core.analysis._analysis import Analysis -from cellprofiler_core.analysis._runner import Runner -from cellprofiler_core.analysis.reply import ImageSetSuccess - -logger = logging.getLogger(__name__) -# logger.addHandler(logging.StreamHandler()) -logger.setLevel(logging.DEBUG) -import six.moves import inspect import numpy import os -import six.moves.queue import tempfile import threading import traceback import unittest import uuid import zmq +import six.moves +import six.moves.queue +from importlib.util import find_spec +import cellprofiler_core.constants.measurement +import cellprofiler_core.utilities.measurement +from cellprofiler_core.analysis._analysis import Analysis +from cellprofiler_core.analysis._runner import Runner +from cellprofiler_core.analysis.reply import ImageSetSuccess import cellprofiler_core.analysis import cellprofiler_core.analysis.request as anarequest import cellprofiler_core.analysis.reply as anareply @@ -39,6 +33,9 @@ import cellprofiler_core.utilities.zmq import cellprofiler_core.utilities.zmq.communicable.reply.upstream_exit + +LOGGER = logging.getLogger(__name__) + IMAGE_NAME = "imagename" OBJECTS_NAME = "objectsname" IMAGE_FEATURE = "imagefeature" @@ -84,7 +81,7 @@ def __exit__(self, type, value, traceback): self.join() def run(self): - logger.info("Client thread starting") + LOGGER.info("Client thread starting") try: self.work_socket = self.zmq_context.socket(zmq.REQ) self.recv_notify_socket = self.zmq_context.socket(zmq.SUB) @@ -116,17 +113,17 @@ def run(self): traceback.print_exc() self.response_queue.put((e, None)) except: - logger.warning("Client thread caught exception", exc_info=True) + LOGGER.warning("Client thread caught exception", exc_info=True) self.start_signal.release() finally: - logger.debug("Client thread exiting") + LOGGER.debug("Client thread exiting") def stop(self): self.keep_going = False self.notify_socket.send(b"Stop") def send(self, req): - logger.debug(" Enqueueing send of %s" % str(type(req))) + LOGGER.debug(" Enqueueing send of %s" % str(type(req))) self.queue.put((self.do_send, req)) self.notify_socket.send(b"Send") return self.recv @@ -143,7 +140,7 @@ def request_work(self): ) def do_send(self, req): - logger.info(" Sending %s" % str(type(req))) + LOGGER.info(" Sending %s" % str(type(req))) cellprofiler_core.utilities.zmq.communicable.Communicable.send( req, self.work_socket ) @@ -156,7 +153,7 @@ def do_send(self, req): if not self.keep_going: raise Exception("Cancelled") if socks.get(self.work_socket, None) == zmq.POLLIN: - logger.debug(" Received response for %s" % str(type(req))) + LOGGER.debug(" Received response for %s" % str(type(req))) return cellprofiler_core.utilities.zmq.communicable.Communicable.recv( self.work_socket ) @@ -164,13 +161,13 @@ def do_send(self, req): self.poller.unregister(self.work_socket) def recv(self): - logger.debug(" Waiting for client thread") + LOGGER.debug(" Waiting for client thread") exception, result = self.response_queue.get() if exception is not None: - logger.debug(" Client thread communicated exception") + LOGGER.debug(" Client thread communicated exception") raise exception else: - logger.debug(" Client thread communicated result") + LOGGER.debug(" Client thread communicated result") return result def listen_for_heartbeat(self, address): @@ -194,7 +191,7 @@ def do_listen_for_heartbeat(self, address): beat = self.keepalive_socket.recv() return beat except: - logger.info("Failed to listen") + LOGGER.info("Failed to listen") finally: self.poller.unregister(self.keepalive_socket) self.keepalive_socket.close() @@ -322,7 +319,7 @@ def check_display_post_run_requests(self, pipeline): self.assertEqual(result.module_num, module.module_num) def test_01_01_start_and_stop(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.make_pipeline_and_measurements_and_start() @@ -338,7 +335,7 @@ def test_01_01_start_and_stop(self): self.assertIsInstance( analysis_finished.measurements, cellprofiler_core.measurement.Measurements ) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -346,7 +343,7 @@ def test_02_01_heartbeat(self): """The heartbeat should send b'KeepAlive' each second. When the analysis is cancelled or finishes we should see b'Stop' as the final transmission on the socket""" - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -379,7 +376,7 @@ def collect_messages(container): assert not spy_thread.is_alive() assert len(messages) assert messages[-1] == NOTIFY_STOP - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -392,12 +389,12 @@ def test_03_01_get_work(self): self.assertSequenceEqual(response.image_set_numbers, (1,)) self.assertFalse(response.worker_runs_post_group) self.assertTrue(response.wants_dictionary) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_03_02_get_work_twice(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -408,12 +405,12 @@ def test_03_02_get_work_twice(self): self.assertIsInstance(response, anareply.Work) response = worker.send(anarequest.Work(worker.analysis_id))() self.assertIsInstance(response, anareply.NoWork) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_03_03_cancel_before_work(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -431,13 +428,13 @@ def test_03_03_cancel_before_work(self): # response, # cellprofiler_core.utilities.zmq.communicable.reply.upstream_exit.BoundaryExited, # ) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) # FIXME: wxPython 4 PR # def test_04_01_pipeline_preferences(self): - # logger.debug("Entering %s" % inspect.getframeinfo(inspect.currentframe()).function) + # LOGGER.debug("Entering %s" % inspect.getframeinfo(inspect.currentframe()).function) # pipeline, m = self.make_pipeline_and_measurements_and_start() # cellprofiler_core.preferences.set_headless() # title_font_name = "Rosewood Std Regular" @@ -477,10 +474,10 @@ def test_03_03_cancel_before_work(self): # self.assertEqual(preferences[cellprofiler_core.preferences.DEFAULT_OUTPUT_DIRECTORY], # cellprofiler_core.preferences.get_default_output_directory()) # - # logger.debug("Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function) + # LOGGER.debug("Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function) def test_04_02_initial_measurements_request(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -522,12 +519,12 @@ def test_04_02_initial_measurements_request(self): numpy.testing.assert_almost_equal(sv, cv) finally: client_measurements.close() - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_03_interaction(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -543,12 +540,12 @@ def test_04_03_interaction(self): reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Interaction) self.assertEqual(reply.hello, "world") - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_04_01_display(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -567,12 +564,12 @@ def test_04_04_01_display(self): reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Ack) self.assertEqual(reply.message, "Gimme Pony") - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_04_02_display_post_group(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -592,12 +589,12 @@ def test_04_04_02_display_post_group(self): reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Ack) self.assertEqual(reply.message, "Gimme Pony") - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_05_exception(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -647,7 +644,7 @@ def test_04_05_exception(self): request.reply(anareply.Ack()) reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Ack) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -661,7 +658,7 @@ def test_05_01_imageset_with_dictionary(self): # request.Work (with spin until WorkReply received) # request.SharedDictionary # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start(nimage_sets=2) @@ -690,12 +687,12 @@ def test_05_01_imageset_with_dictionary(self): self.assertCountEqual(list(ed.keys()), list(d.keys())) for k in list(ed.keys()): numpy.testing.assert_almost_equal(ed[k], d[k]) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_05_02_groups(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start( @@ -716,7 +713,7 @@ def test_05_02_groups(self): self.assertSequenceEqual(response.image_set_numbers, [3, 4]) self.assertTrue(response.worker_runs_post_group) self.assertFalse(response.wants_dictionary) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -725,7 +722,7 @@ def test_06_01_single_imageset(self): # Test a full cycle of analysis with an image set list # with a single image set # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -801,7 +798,7 @@ def test_06_01_single_imageset(self): def test_06_02_test_three_imagesets(self): # Test an analysis of three imagesets # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -907,7 +904,7 @@ def test_06_02_test_three_imagesets(self): def test_06_03_test_grouped_imagesets(self): # Test an analysis of four imagesets in two groups # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -998,7 +995,7 @@ def test_06_03_test_grouped_imagesets(self): def test_06_04_test_restart(self): # Test a restart of an analysis # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -1121,7 +1118,7 @@ def test_06_05_test_grouped_restart(self): # Test an analysis of four imagesets in two groups with all but one # complete. # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -1202,7 +1199,7 @@ def test_06_06_relationships(self): # # Test a transfer of the relationships table. # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -1316,7 +1313,7 @@ def test_06_07_worker_cancel(self): # # Test worker sending AnalysisCancelRequest # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py index fac6dc2a..94ac78a6 100644 --- a/tests/modules/__init__.py +++ b/tests/modules/__init__.py @@ -1,19 +1,18 @@ import os import tempfile - -import cellprofiler_core.utilities.legacy - -from cellprofiler_core.bioformats.omexml import PT_UINT16 -from cellprofiler_core.bioformats.formatwriter import write_image import logging import functools import hashlib - -logger = logging.getLogger(__name__) import numpy import unittest from urllib.request import URLopener +import cellprofiler_core.utilities.legacy +from cellprofiler_core.bioformats.omexml import PT_UINT16 +from cellprofiler_core.bioformats.formatwriter import write_image + +LOGGER = logging.getLogger(__name__) + __temp_example_images_folder = None __temp_test_images_folder = None @@ -38,7 +37,7 @@ def example_images_directory(): return path if __temp_example_images_folder is None: __temp_example_images_folder = tempfile.mkdtemp(prefix="cp_exampleimages") - logger.warning( + LOGGER.warning( "Creating temporary folder %s for example images" % __temp_example_images_folder ) @@ -60,7 +59,7 @@ def testimages_directory(): return path if __temp_test_images_folder is None: __temp_test_images_folder = tempfile.mkdtemp(prefix="cp_testimages") - logger.warning( + LOGGER.warning( "Creating temporary folder %s for test images" % __temp_test_images_folder ) return __temp_test_images_folder diff --git a/tests/workspace/test_workspace.py b/tests/workspace/test_workspace.py index e9f12b36..7b050ce3 100644 --- a/tests/workspace/test_workspace.py +++ b/tests/workspace/test_workspace.py @@ -10,8 +10,8 @@ import cellprofiler_core.workspace from cellprofiler_core.utilities.hdf5_dict import FILE_LIST_GROUP, TOP_LEVEL_GROUP_NAME -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class TestWorkspace: def setup_method(self): @@ -22,7 +22,7 @@ def teardown_method(self): try: os.remove(path) except OSError: - logger.warning("Failed to close file %s" % path, exc_info=1) + LOGGER.warning("Failed to close file %s" % path, exc_info=1) def make_workspace_file(self): """Make a very basic workspace file""" From bef16a90871ed097fa3862711986e781119355ea Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Wed, 30 Nov 2022 18:41:53 -0500 Subject: [PATCH 30/52] Logging Refactor - pass parent debug level to analysis worker - format worker log mesage with log level - have parent parse and log with corresponding log level - resolves CellProfiler/CellProfiler#4651 --- cellprofiler_core/analysis/_runner.py | 19 ++++++++++++++++--- cellprofiler_core/worker/__init__.py | 6 +++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cellprofiler_core/analysis/_runner.py b/cellprofiler_core/analysis/_runner.py index 68dd40db..97521f9c 100644 --- a/cellprofiler_core/analysis/_runner.py +++ b/cellprofiler_core/analysis/_runner.py @@ -8,6 +8,7 @@ import tempfile import threading from typing import List, Any +import re import numpy import psutil @@ -659,7 +660,9 @@ def start_workers(self, num=None): "--conserve-memory", str(get_conserve_memory()), "--always-continue", - str(get_always_continue()) + str(get_always_continue()), + "--log-level", + str(logging.root.level) ] # start workers @@ -703,8 +706,18 @@ def run_logger(workR, widx): line = line.decode("utf-8") if not line: break - LOGGER.info("Worker %d: %s", widx, line.rstrip()) - except: + log_msg_match = re.match(fr"{workR.pid}\|(10|20|30|40|50)\|(.*)", line) + if log_msg_match: + levelno = int(log_msg_match.group(1)) + msg = log_msg_match.group(2) + else: + levelno = 20 + msg = line + + LOGGER.log(levelno, "\n\r\t[Worker %d] %s", widx, msg.rstrip()) + + except Exception as e: + LOGGER.exception(e) break start_daemon_thread( diff --git a/cellprofiler_core/worker/__init__.py b/cellprofiler_core/worker/__init__.py index a587c8d2..793caf05 100644 --- a/cellprofiler_core/worker/__init__.py +++ b/cellprofiler_core/worker/__init__.py @@ -83,6 +83,7 @@ def aw_parse_args(): parser.add_option( "--log-level", dest="log_level", + type="int", help="Logging level for logger: DEBUG, INFO, WARNING, ERROR", default=os.environ.get(AW_LOG_LEVEL, logging.INFO), ) @@ -125,7 +126,10 @@ def aw_parse_args(): logging.root.setLevel(options.log_level) if len(logging.root.handlers) == 0: - logging.root.addHandler(logging.StreamHandler()) + stream_handler = logging.StreamHandler() + fmt = logging.Formatter("%(process)d|%(levelno)s|%(name)s::%(funcName)s : %(message)s") + stream_handler.setFormatter(fmt) + logging.root.addHandler(stream_handler) if not options.work_server_address and options.work_server_address and \ options.analysis_id: From a2264b12cf3fa70c65219a7326ccb840eee2ecb1 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Wed, 30 Nov 2022 19:29:42 -0500 Subject: [PATCH 31/52] Refactor Logging couldn't figure out why we're logging out a traceback so I disabled it --- cellprofiler_core/measurement/_measurements.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cellprofiler_core/measurement/_measurements.py b/cellprofiler_core/measurement/_measurements.py index 8570e624..eba52ce2 100644 --- a/cellprofiler_core/measurement/_measurements.py +++ b/cellprofiler_core/measurement/_measurements.py @@ -108,11 +108,11 @@ def __init__( elif filename is None: fd, filename = make_temporary_file() is_temporary = True - import traceback - LOGGER.debug("Created temporary file %s" % filename) - for frame in traceback.extract_stack(): - LOGGER.debug("{}: ({} {}): {}".format(*frame)) + + # import traceback + # for frame in traceback.extract_stack(): + # LOGGER.debug("{}: ({} {}): {}".format(*frame)) else: is_temporary = False From 1261c2426a24f6d9478ab5f5fa71ac2ae1f21c3f Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 1 Dec 2022 17:47:54 -0500 Subject: [PATCH 32/52] Logging Refactor some cleanup --- cellprofiler_core/analysis/_runner.py | 2 +- cellprofiler_core/utilities/hdf5_dict.py | 2 +- cellprofiler_core/worker/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cellprofiler_core/analysis/_runner.py b/cellprofiler_core/analysis/_runner.py index 97521f9c..8dbf450e 100644 --- a/cellprofiler_core/analysis/_runner.py +++ b/cellprofiler_core/analysis/_runner.py @@ -714,7 +714,7 @@ def run_logger(workR, widx): levelno = 20 msg = line - LOGGER.log(levelno, "\n\r\t[Worker %d] %s", widx, msg.rstrip()) + LOGGER.log(levelno, "\n\r [Worker %d (%d)] %s", widx, workR.pid, msg.rstrip()) except Exception as e: LOGGER.exception(e) diff --git a/cellprofiler_core/utilities/hdf5_dict.py b/cellprofiler_core/utilities/hdf5_dict.py index 6ec372bb..0d519e9c 100644 --- a/cellprofiler_core/utilities/hdf5_dict.py +++ b/cellprofiler_core/utilities/hdf5_dict.py @@ -365,7 +365,7 @@ def __init__( def __del__(self): LOGGER.debug( - "HDF5Dict.__del__(): %s, temporary=%s", self.filename, self.is_temporary + "%s, temporary=%s", self.filename, self.is_temporary ) self.close() diff --git a/cellprofiler_core/worker/__init__.py b/cellprofiler_core/worker/__init__.py index 793caf05..3f706b88 100644 --- a/cellprofiler_core/worker/__init__.py +++ b/cellprofiler_core/worker/__init__.py @@ -127,7 +127,7 @@ def aw_parse_args(): logging.root.setLevel(options.log_level) if len(logging.root.handlers) == 0: stream_handler = logging.StreamHandler() - fmt = logging.Formatter("%(process)d|%(levelno)s|%(name)s::%(funcName)s : %(message)s") + fmt = logging.Formatter("%(process)d|%(levelno)s|%(name)s::%(funcName)s: %(message)s") stream_handler.setFormatter(fmt) logging.root.addHandler(stream_handler) From 00cbf75c333f87fed95b2cca64d39a5d239c05b3 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 1 Dec 2022 17:55:20 -0500 Subject: [PATCH 33/52] Fix volume reading ensure range is passed into itertools.product rather than a scalar --- cellprofiler_core/readers/bioformats_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 62a20c9d..07c2f38c 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -104,7 +104,7 @@ def read_volume(self, z_range = [z] if t is None: if z_len > 1: - t_range = 1 + t_range = range(1) else: t_range = range(bf_reader.getSizeT()) else: From 1f0937962218340639f88e89a0cc770f305045a3 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Fri, 2 Dec 2022 12:12:54 -0500 Subject: [PATCH 34/52] Disable omero related stuff until CellProfiler/CellProfiler#4684 is resolved, anything omero related is disabled (either commented out or unreachable) --- cellprofiler_core/bioformats/formatreader.py | 8 +++----- cellprofiler_core/constants/image.py | 4 +++- cellprofiler_core/pipeline/_pipeline.py | 3 ++- cellprofiler_core/readers/bioformats_reader.py | 4 +++- cellprofiler_core/worker/_worker.py | 5 +++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cellprofiler_core/bioformats/formatreader.py b/cellprofiler_core/bioformats/formatreader.py index d7c00c92..3b2e8b38 100644 --- a/cellprofiler_core/bioformats/formatreader.py +++ b/cellprofiler_core/bioformats/formatreader.py @@ -1,4 +1,4 @@ -#CTR FIXME: All of the below should be removed, deleted, perished, annihilated. +#TODO: unimplemented until CellProfiler/CellProfiler#4684 is resolved K_OMERO_SERVER = None K_OMERO_PORT = None @@ -7,12 +7,10 @@ K_OMERO_CONFIG_FILE = None def clear_image_reader_cache(): - pass - #$raise RuntimeError("unimplemented") + raise RuntimeError("unimplemented") def set_omero_login_hook(omero_login): - pass - #raise RuntimeError("unimplemented") + raise RuntimeError("unimplemented") def get_omero_credentials(): raise RuntimeError("unimplemented") diff --git a/cellprofiler_core/constants/image.py b/cellprofiler_core/constants/image.py index 094fbc68..71c8a461 100644 --- a/cellprofiler_core/constants/image.py +++ b/cellprofiler_core/constants/image.py @@ -152,7 +152,9 @@ SUB_ALL = "All" SUB_SOME = "Some" FILE_SCHEME = "file:" -PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "omero", "s3","gs") +#TODO: disabled until CellProfiler/CellProfiler#4684 is resolved +# PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "omero", "s3","gs") +PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "s3","gs") CT_GRAYSCALE = "Grayscale" CT_COLOR = "Color" diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index de7c767b..b948d899 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -84,6 +84,7 @@ from ..constants.workspace import DISPOSITION_CANCEL from ..constants.workspace import DISPOSITION_PAUSE from ..constants.workspace import DISPOSITION_SKIP +from ..constants.image import PASSTHROUGH_SCHEMES from ..constants.modules.metadata import X_AUTOMATIC_EXTRACTION from ..image import ImageSetList from ..measurement import Measurements @@ -2071,7 +2072,7 @@ def read_file_list(self, path_or_fd, add_undo=True): self.read_file_list(fd, add_undo=add_undo) elif any( pathname.startswith(protocol) - for protocol in ("http", "https", "ftp", "omero", "s3", "gs") + for protocol in PASSTHROUGH_SCHEMES ): with urllib.request.urlopen(pathname) as response: data = response.read().decode("utf-8").splitlines() diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 5bed9d09..4d0e569a 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -21,7 +21,9 @@ # bioformats returns 2 for these, imageio reader returns 3 SUPPORTED_EXTENSIONS = {'.tiff', '.tif', '.ome.tif', '.ome.tiff'} SEMI_SUPPORTED_EXTENSIONS = BIOFORMATS_IMAGE_EXTENSIONS -SUPPORTED_SCHEMES = {'file', 'http', 'https', 'ftp', 'ftps', 'omero', 's3'} +#TODO: disabled until CellProfiler/CellProfiler#4684 is resolved +# SUPPORTED_SCHEMES = {'file', 'http', 'https', 'ftp', 'ftps', 'omero', 's3'} +SUPPORTED_SCHEMES = {'file', 'http', 'https', 'ftp', 'ftps', 's3'} class BioformatsReader(Reader): """ diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index 3eb0995a..3e4801fa 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -40,7 +40,6 @@ class Worker: """ def __init__(self, context, analysis_id, work_request_address, keepalive_address, with_stop_run_loop=True): - from ..bioformats.formatreader import set_omero_login_hook self.context = context self.work_request_address = work_request_address @@ -52,7 +51,9 @@ def __init__(self, context, analysis_id, work_request_address, keepalive_address self.preferences = None self.initial_measurements = None - set_omero_login_hook(self.omero_login_handler) + #TODO: disabled until CellProfiler/CellProfiler#4684 is resolved + # from ..bioformats.formatreader import set_omero_login_hook + # set_omero_login_hook(self.omero_login_handler) def __enter__(self): # pipeline listener object From 535cd1654863f4a4b12283ae82cfb3548cb1aa44 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Fri, 2 Dec 2022 15:28:38 -0500 Subject: [PATCH 35/52] Logging formatting --- cellprofiler_core/pipeline/_pipeline.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index b948d899..f1ab2f57 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -478,16 +478,16 @@ def validate_pipeline_version(self, current_version, git_hash, pipeline_version) if (not get_headless()) and pipeline_version < current_version: if git_hash is not None: message = ( - "Your pipeline was saved using an old version\n" - "of CellProfiler (rev {}{}).\n" - "The current version of CellProfiler can load\n" - "and run this pipeline, but if you make changes\n" - "to it and save, the older version of CellProfiler\n" - "(perhaps the version your collaborator has?) may\n" - "not be able to load it.\n\n" - "You can ignore this warning if you do not plan to save\n" - "this pipeline or if you will only use it with this or\n" - "later versions of CellProfiler." + "\n\tYour pipeline was saved using an old version\n" + "\tof CellProfiler (rev {}{}).\n" + "\tThe current version of CellProfiler can load\n" + "\tand run this pipeline, but if you make changes\n" + "\tto it and save, the older version of CellProfiler\n" + "\t(perhaps the version your collaborator has?) may\n" + "\tnot be able to load it.\n\n" + "\tYou can ignore this warning if you do not plan to save\n" + "\tthis pipeline or if you will only use it with this or\n" + "\tlater versions of CellProfiler." ).format(git_hash, pipeline_date) LOGGER.warning(message) else: @@ -637,8 +637,8 @@ def setup_module( for line in value ] except Exception as e: - print( - f"Error during notes decoding - {e} \nSome characters may have been lost" + LOGGER.error( + f"Error during notes decoding\n\t{e}\n\tSome characters may have been lost" ) else: value = eval(value) From 0a91bcfb33999c487bdccdeb4ea01c0954846275 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Fri, 2 Dec 2022 17:08:30 -0500 Subject: [PATCH 36/52] Fix plateviewer --- cellprofiler_core/readers/bioformats_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 4d0e569a..aec9554e 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -60,8 +60,8 @@ def read(self, series=None, index=None, c=None, - z=None, - t=None, + z=0, + t=0, rescale=True, xywh=None, wants_max_intensity=False, From 79d2a866fb5e4578269c8389b9f1cfba9a937f4a Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Fri, 2 Dec 2022 21:12:31 -0500 Subject: [PATCH 37/52] Writer docs --- cellprofiler_core/bioformats/formatwriter.py | 17 +++++++++++------ cellprofiler_core/measurement/_measurements.py | 2 +- cellprofiler_core/modules/metadata.py | 3 ++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py index 757f1f86..99f3f123 100644 --- a/cellprofiler_core/bioformats/formatwriter.py +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -23,15 +23,22 @@ def write_image( size_t, channel_names ): - ImageWriter = jimport("loci.formats.ImageWriter") + # https://www.javadoc.io/doc/org.openmicroscopy/ome-common/5.3.2/loci/common/services/ServiceFactory.html OMEXMLServiceFactory = jimport("loci.common.services.ServiceFactory") - DimensionsOrder = jimport("ome.xml.model.enums.DimensionOrder") - PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger") - PixelType = jimport("ome.xml.model.enums.PixelType") + # https://javadoc.scijava.org/Bio-Formats/loci/formats/services/OMEXMLService.html OMEXMLService = jimport("loci.formats.services.OMEXMLService") + # https://javadoc.scijava.org/Bio-Formats/loci/formats/ImageWriter.html + ImageWriter = jimport("loci.formats.ImageWriter") + # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/IMetadata.html IMetadata = jimport("loci.formats.meta.IMetadata") + DimensionsOrder = jimport("ome.xml.model.enums.DimensionOrder") + PixelType = jimport("ome.xml.model.enums.PixelType") + PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger") + omexml_service = OMEXMLServiceFactory().getInstance(OMEXMLService) + # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/OMEXMLMetadata.html + # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/MetadataStore.html metadata = omexml_service.createOMEXMLMetadata() metadata.createRoot() @@ -55,8 +62,6 @@ def write_image( elif size_c > 1: # meta.channel_count = size_c <- cant find metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0) - # FIXME do this per channel - metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, 0) omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2])) metadata.setImageID("Image:0", 0) diff --git a/cellprofiler_core/measurement/_measurements.py b/cellprofiler_core/measurement/_measurements.py index eba52ce2..f24d0d95 100644 --- a/cellprofiler_core/measurement/_measurements.py +++ b/cellprofiler_core/measurement/_measurements.py @@ -1021,7 +1021,7 @@ def apply_metadata(self, pattern, image_set_number=None): value = self[ IMAGE, mname, image_set_number, ] - if value > max_value: + if value and value > max_value: max_value = value result += str(max_value) else: diff --git a/cellprofiler_core/modules/metadata.py b/cellprofiler_core/modules/metadata.py index 976ac173..c5805674 100644 --- a/cellprofiler_core/modules/metadata.py +++ b/cellprofiler_core/modules/metadata.py @@ -891,7 +891,8 @@ def run_extraction(self, file_objects): file_object.add_metadata(candidate_dict) break if not valid: - print(f"No matching metadata found for {file_object.filename}") + LOGGER.info(f"No matching metadata found for {file_object.filename}") + break pass else: raise NotImplementedError(f"Invalid extraction method '{group.extraction_method}'") From d28886cb1c89ddebbb8768d5b0d40ada4fa4c1b3 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Wed, 7 Dec 2022 19:21:54 -0500 Subject: [PATCH 38/52] Remove unimplemented cache clearing --- .../readers/bioformats_reader.py | 20 +------------------ cellprofiler_core/worker/_worker.py | 3 +-- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index aec9554e..e3ccbdd5 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -16,8 +16,6 @@ LOGGER = logging.getLogger(__name__) -LOGGER.info("HELLO! It's a-ME, bioformats_reader. I just added Bio-Formats endpoint.") - # bioformats returns 2 for these, imageio reader returns 3 SUPPORTED_EXTENSIONS = {'.tiff', '.tif', '.ome.tif', '.ome.tiff'} SEMI_SUPPORTED_EXTENSIONS = BIOFORMATS_IMAGE_EXTENSIONS @@ -83,7 +81,6 @@ def read(self, :param channel_names: provide the channel names for the OME metadata :param XYWH: a (x, y, w, h) tuple """ - LOGGER.info("--> bioformats_reader.read BEGINS") self._ensure_file_open() FormatTools = jimport("loci.formats.FormatTools") @@ -99,7 +96,6 @@ def read(self, openBytes_func = self._reader.openBytes width, height = self._reader.getSizeX(), self._reader.getSizeY() - LOGGER.info("--> bioformats_reader.read: Decided openBytes func") # FIXME instead of np.frombuffer use scyjava.to_python, ideally that wraps memory pixel_type = self._reader.getPixelType() little_endian = self._reader.isLittleEndian() @@ -134,7 +130,6 @@ def read(self, except: LOGGER.warning("WARNING: failed to get MaxSampleValue for image. Intensities may be improperly scaled.") if index is not None: - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) index!=None CASE") image = np.frombuffer(openBytes_func(index), dtype) if len(image) / height / width in (3,4): n_channels = int(len(image) / height / width) @@ -147,18 +142,15 @@ def read(self, image.shape = (height, width) elif self._reader.isRGB() and self._reader.isInterleaved(): index = self._reader.getIndex(z,0,t) - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGB INTERLEAVED CASE") image = np.frombuffer(openBytes_func(index), dtype) image.shape = (height, width, self._reader.getSizeC()) if image.shape[2] > 3: image = image[:, :, :3] elif c is not None and self._reader.getRGBChannelCount() == 1: index = self._reader.getIndex(z,c,t) - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) RGBChannelCount==1 CASE") image = np.frombuffer(openBytes_func(index), dtype) image.shape = (height, width) elif self._reader.getRGBChannelCount() > 1: - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes RGBChannelCount>1 CASE") n_planes = self._reader.getRGBChannelCount() rdr = ChannelSeparator(self._reader) planes = [ @@ -177,7 +169,6 @@ def read(self, image = np.dstack(planes) image.shape=(height, width, 3) elif self._reader.getSizeC() > 1: - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes SizeC>1 CASE") images = [ np.frombuffer(openBytes_func(self._reader.getIndex(z,i,t)), dtype) for i in range(self._reader.getSizeC())] @@ -200,7 +191,6 @@ def read(self, # a monochrome RGB image # index = self._reader.getIndex(z,0,t) - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) INDEXED CASE") image = np.frombuffer(openBytes_func(index),dtype) lut = None if pixel_type in (FormatTools.INT16, FormatTools.UINT16): @@ -215,7 +205,6 @@ def read(self, image = lut[image, :] else: index = self._reader.getIndex(z,0,t) - LOGGER.info(f"--> bioformats_reader.read: Calling openBytes({index}) ") image = np.frombuffer(openBytes_func(index),dtype) image.shape = (height,width) @@ -290,12 +279,11 @@ def supports_format(cls, image_file, allow_open=False, volume=False): The volume parameter specifies whether the reader will need to return a 3D array. .""" try: - LOGGER.info(f"--> bioformats_reader.supports_format: isThisType({image_file.path}, allow_open={allow_open})") ImageReader = jimport("loci.formats.ImageReader") is_this_type = ImageReader().isThisType(image_file.path, allow_open) - LOGGER.info(f"--> bioformats_reader.supports_format: DRUMROLL... is it compatible? ... {is_this_type}") except Exception as ex: LOGGER.error(ex) + return -1 if image_file.scheme not in SUPPORTED_SCHEMES: return -1 @@ -308,12 +296,6 @@ def supports_format(cls, image_file, allow_open=False, volume=False): return 3 return -1 - @classmethod - def clear_cached_readers(cls): - #CTR FIXME: Do we need this? - #clear_image_reader_cache() - pass - def close(self): # If your reader opens a file, this needs to release any active lock, if self._reader is not None and scyjava.jvm_started(): diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index 3e4801fa..f694389e 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -95,8 +95,7 @@ def __exit__(self, type, value, tb): self.worker.exit_thread() def enter_thread(self): - # CTR FIXME: Do we still need this function? - pass + LOGGER.debug("Entering worker thread") def exit_thread(self): from cellprofiler_core.constants.reader import ALL_READERS From a5c55507184e1b37be3eb2778b954f51241af9b4 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 8 Dec 2022 11:34:47 -0500 Subject: [PATCH 39/52] Remove last python-bioformats remnants --- .../image/abstract_image/file/url/_objects_image.py | 2 +- cellprofiler_core/worker/_worker.py | 2 +- tests/conftest.py | 4 ---- tests/modules/__init__.py | 9 ++++++--- tests/modules/test_loaddata.py | 9 ++------- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cellprofiler_core/image/abstract_image/file/url/_objects_image.py b/cellprofiler_core/image/abstract_image/file/url/_objects_image.py index 83acaa49..2d177ae3 100644 --- a/cellprofiler_core/image/abstract_image/file/url/_objects_image.py +++ b/cellprofiler_core/image/abstract_image/file/url/_objects_image.py @@ -53,7 +53,7 @@ def provide_image(self, image_set): planes = [] offset = 0 for i, index in enumerate(indexes): - properties["index"] = str(index) + properties["index"] = index if self.series is not None: if numpy.isscalar(self.series): properties["series"] = self.series diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index f694389e..7db7768b 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -359,7 +359,7 @@ def post_group_display_handler(self, module, display_data, image_set_number): def omero_login_handler(self): """Handle requests for an Omero login""" - from bioformats.formatreader import use_omero_credentials + from cellprofiler_core.bioformats.formatreader import use_omero_credentials req = OmeroLogin(self.current_analysis_id) rep = self.send(req) diff --git a/tests/conftest.py b/tests/conftest.py index 6525c127..e4efa53a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,3 @@ -import bioformats.formatreader - import cellprofiler_core.preferences import cellprofiler_core.utilities.java @@ -9,8 +7,6 @@ def pytest_sessionstart(session): def pytest_sessionfinish(session, exitstatus): - bioformats.formatreader.clear_image_reader_cache() - cellprofiler_core.utilities.java.stop_java() return exitstatus diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py index 94ac78a6..d7a0ca11 100644 --- a/tests/modules/__init__.py +++ b/tests/modules/__init__.py @@ -6,10 +6,9 @@ import numpy import unittest from urllib.request import URLopener +import skimage.io import cellprofiler_core.utilities.legacy -from cellprofiler_core.bioformats.omexml import PT_UINT16 -from cellprofiler_core.bioformats.formatwriter import write_image LOGGER = logging.getLogger(__name__) @@ -124,7 +123,11 @@ def make_12_bit_image(folder, filename, shape): if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) - write_image(path, img, PT_UINT16) + if len(shape) > 2: + skimage.io.imsave(path, numpy.transpose(img, (2,0,1)), imagej=True) + else: + skimage.io.imsave(path, img) + # # Now go through the file and find the TIF bits per sample IFD (#258) and # change it from 16 to 12. diff --git a/tests/modules/test_loaddata.py b/tests/modules/test_loaddata.py index f9d7b918..c572f4c5 100644 --- a/tests/modules/test_loaddata.py +++ b/tests/modules/test_loaddata.py @@ -4,10 +4,9 @@ import re import tempfile -import bioformats -import bioformats.formatreader import numpy import pytest +import skimage.io from cellprofiler_core.constants.measurement import COLTYPE_FLOAT from cellprofiler_core.constants.measurement import COLTYPE_INTEGER @@ -712,8 +711,6 @@ def test_scaling(): """Test loading an image scaled and unscaled""" folder = "loaddata" file_name = "1-162hrh2ax2.tif" - if not cellprofiler_core.utilities.java.JAVA_STARTED: - start_java() path = tests.modules.make_12_bit_image(folder, file_name, (22, 18)) csv_text = ( "Image_PathName_MyFile,Image_FileName_MyFile\n" "%s,%s\n" % os.path.split(path) @@ -749,8 +746,7 @@ def test_load_objects(): r.seed(1101) labels = r.randint(0, 10, size=(30, 20)).astype(numpy.uint8) handle, name = tempfile.mkstemp(".png") - start_java() - bioformats.write_image(name, labels, bioformats.PT_UINT8) + skimage.io.imsave(name, labels) os.close(handle) png_path, png_file = os.path.split(name) sbs_dir = os.path.join(get_data_directory(), "ExampleSBSImages") @@ -798,7 +794,6 @@ def test_load_objects(): value = measurements.get_current_measurement(OBJECTS_NAME, feature) assert len(value) == 9 finally: - bioformats.formatreader.clear_image_reader_cache() os.remove(name) os.remove(csv_name) From 03eabf75b9f3520ac74f29a2cc2416278ae8e6b6 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 8 Dec 2022 14:21:29 -0500 Subject: [PATCH 40/52] Remove is_this_type check, not necessary --- cellprofiler_core/readers/bioformats_reader.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index e3ccbdd5..1681c41b 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -278,13 +278,6 @@ def supports_format(cls, image_file, allow_open=False, volume=False): The volume parameter specifies whether the reader will need to return a 3D array. .""" - try: - ImageReader = jimport("loci.formats.ImageReader") - is_this_type = ImageReader().isThisType(image_file.path, allow_open) - except Exception as ex: - LOGGER.error(ex) - return -1 - if image_file.scheme not in SUPPORTED_SCHEMES: return -1 if image_file.scheme == 'omero': From 153d4b479082a4179c72619c4fe25894c310607a Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Fri, 9 Dec 2022 10:25:10 -0500 Subject: [PATCH 41/52] Add ets extension --- cellprofiler_core/constants/image.py | 2 +- cellprofiler_core/measurement/_measurements.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cellprofiler_core/constants/image.py b/cellprofiler_core/constants/image.py index 71c8a461..385f0dd1 100644 --- a/cellprofiler_core/constants/image.py +++ b/cellprofiler_core/constants/image.py @@ -107,7 +107,7 @@ ".stk", ".stp", ".svs", ".sxm", ".tc.", ".tf2", ".tf8", ".tfr", ".tga", ".tif", ".tiff", ".tnb", ".top", ".txt", ".v", ".vff", ".vms", ".vsi", ".vws", ".wat", ".wlz", ".wpi", ".xdce", ".xml", ".xqd", ".xqf", ".xv", - ".xys", ".zfp", ".zfr", ".zvi" + ".xys", ".zfp", ".zfr", ".zvi", ".ets" ) DISALLOWED_BIOFORMATS_EXTENSIONS = { diff --git a/cellprofiler_core/measurement/_measurements.py b/cellprofiler_core/measurement/_measurements.py index f24d0d95..4f7256ad 100644 --- a/cellprofiler_core/measurement/_measurements.py +++ b/cellprofiler_core/measurement/_measurements.py @@ -110,10 +110,6 @@ def __init__( is_temporary = True LOGGER.debug("Created temporary file %s" % filename) - # import traceback - # for frame in traceback.extract_stack(): - # LOGGER.debug("{}: ({} {}): {}".format(*frame)) - else: is_temporary = False if isinstance(copy, Measurements): From f8e0a1ed89ad1415b3ca0d3c1200c34caf498ae3 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Fri, 9 Dec 2022 18:36:22 -0500 Subject: [PATCH 42/52] Fix Disappearing Metadata fix jamboree issue, where Metadata module options disapper upon loading cppipe that made reference to a nonexistent input folder --- cellprofiler_core/module/_module.py | 4 ++-- cellprofiler_core/modules/metadata.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellprofiler_core/module/_module.py b/cellprofiler_core/module/_module.py index 827c4726..0403e056 100644 --- a/cellprofiler_core/module/_module.py +++ b/cellprofiler_core/module/_module.py @@ -400,8 +400,8 @@ def test_valid(self, pipeline): for setting in self.visible_settings(): setting.test_valid(pipeline) self.validate_module(pipeline) - except ValidationError as instance: - raise instance + except ValidationError as ve: + raise ve except Exception as e: raise ValidationError( "Exception in cpmodule.test_valid %s" % e, self.visible_settings()[0] diff --git a/cellprofiler_core/modules/metadata.py b/cellprofiler_core/modules/metadata.py index c5805674..dd7351eb 100644 --- a/cellprofiler_core/modules/metadata.py +++ b/cellprofiler_core/modules/metadata.py @@ -698,7 +698,7 @@ def get_csv_header(self, group): group.imported_metadata_col_names = rdr.fieldnames fd.close() except Exception as e: - print("Error while decoding CSV:", e) + LOGGER.error(f"Error while decoding CSV - {csv_path}: {e.strerror}") return None return group.imported_metadata_col_names @@ -786,7 +786,7 @@ def visible_settings(self): result += [self.add_extraction_method_button] try: has_keys = len(self.get_dt_metadata_keys()) > 0 - except Exception: + except Exception as e: has_keys = False if has_keys: result += [self.dtc_divider, self.data_type_choice] @@ -1072,7 +1072,7 @@ def get_metadata_keys(self): self.compile_regex(extract_group) keys.update(extract_group.regex_pattern.groupindex.keys()) elif extract_group.extraction_method == X_IMPORTED_EXTRACTION: - keys.update(self.get_csv_header(extract_group)) + keys.update(self.get_csv_header(extract_group) or '') if FTR_WELL not in keys and any(k in keys for k in ROW_KEYS) and any(k in keys for k in COL_KEYS): keys.add(FTR_WELL) return keys @@ -1083,7 +1083,7 @@ def get_dt_metadata_keys(self): """ return list( filter( - (lambda k: k not in self.NUMERIC_DATA_TYPES), self.get_metadata_keys() + (lambda k: k not in self.NUMERIC_DATA_TYPES), self.get_metadata_keys() or [] ) ) From 65ce2b4f8bb28cecedc02a76ac522e9f0695576c Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 15 Dec 2022 18:33:58 -0500 Subject: [PATCH 43/52] Remove unused unicode flags --- cellprofiler_core/pipeline/_pipeline.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index f1ab2f57..6c4b2a5d 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -430,12 +430,6 @@ def validate_pipeline_file(self, fd, module_count): version, NATIVE_VERSION ) ) - elif 1 < version < 4: - do_deprecated_utf16_decode = True - elif version == 4: - do_utf16_decode = True - elif version == 5: - pass elif kwd in (H_SVN_REVISION, H_DATE_REVISION,): pipeline_version = int(value) elif kwd == H_MODULE_COUNT: @@ -644,9 +638,7 @@ def setup_module( value = eval(value) if attribute in skip_attributes: continue - # En/decode needed to read example cppipe format - # TODO: remove en/decode when example cppipe no longer has \x__ characters - # value = eval(value.encode().decode("unicode_escape")) + if attribute == "variable_revision_number": variable_revision_number = value else: From d74a837be736235d39ab818e6c7312d8a8da6786 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 15 Dec 2022 18:36:22 -0500 Subject: [PATCH 44/52] Remove unused unicode flags --- cellprofiler_core/pipeline/_pipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index 6c4b2a5d..ea4deb2e 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -402,8 +402,6 @@ def parse_pipeline(self, fd, raise_on_error, count=sys.maxsize): def validate_pipeline_file(self, fd, module_count): version = NATIVE_VERSION - do_deprecated_utf16_decode = False - do_utf16_decode = False has_image_plane_details = False git_hash = None pipeline_version = __version__ From 34633495e5f4c26f6ac01b8cee095f52fc141e69 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 15 Dec 2022 20:20:45 -0500 Subject: [PATCH 45/52] [todo] Remove use of eval --- cellprofiler_core/pipeline/_pipeline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index ea4deb2e..dbb046ea 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -449,6 +449,8 @@ def validate_pipeline_file(self, fd, module_count): return git_hash, has_image_plane_details, module_count, pipeline_version def validate_pipeline_version(self, current_version, git_hash, pipeline_version): + # if "pipeline_version" is an actual dave revision, ie unicode time + # at some point this changed to CellProfiler version (e.g. CP 4.2.4 would be 400) if 20080101000000 < pipeline_version < 30080101000000: # being optomistic... a millenium should be OK, no? second, minute, hour, day, month = [ @@ -605,6 +607,8 @@ def setup_module( # make batch_state decodable from text pipelines # NOTE, MAGIC HERE: These variables are **necessary**, even though they # aren't used anywhere obvious. Removing them **will** break these unit tests. + # TODO: NG - MAGIC here is caused by the use of eval, which is evil, and should + # be totally expunged here and everywhere else it is used array = numpy.array uint8 = numpy.uint8 for a in attribute_strings: @@ -619,7 +623,7 @@ def setup_module( if attribute == "notes": try: value = eval(value) - except SyntaxError: + except SyntaxError as e: value = value[1:-1].replace('"', "").replace("'", "").split(",") if any([chr(226) in line for line in value]): # There were some unusual UTF-8 characters present, let's try to fix them. From f211ea548999e6adb9fdd504082bc427d0a645fc Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Mon, 19 Dec 2022 14:40:44 -0500 Subject: [PATCH 46/52] Deprecate batch_state parsing and associated magic --- cellprofiler_core/pipeline/_pipeline.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index dbb046ea..f5e742f8 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -15,6 +15,7 @@ import urllib.request import uuid import weakref +from ast import literal_eval import numpy @@ -604,13 +605,7 @@ def setup_module( attribute_string = attribute_string.replace("dtype='|S", "dtype='S") attribute_strings = attribute_string[1:-1].split("|") variable_revision_number = None - # make batch_state decodable from text pipelines - # NOTE, MAGIC HERE: These variables are **necessary**, even though they - # aren't used anywhere obvious. Removing them **will** break these unit tests. - # TODO: NG - MAGIC here is caused by the use of eval, which is evil, and should - # be totally expunged here and everywhere else it is used - array = numpy.array - uint8 = numpy.uint8 + for a in attribute_strings: if a.isascii(): a = a.encode("utf-8").decode("unicode_escape") @@ -622,7 +617,7 @@ def setup_module( attribute, value = a.split(":", 1) if attribute == "notes": try: - value = eval(value) + value = literal_eval(value) except SyntaxError as e: value = value[1:-1].replace('"', "").replace("'", "").split(",") if any([chr(226) in line for line in value]): @@ -636,8 +631,10 @@ def setup_module( LOGGER.error( f"Error during notes decoding\n\t{e}\n\tSome characters may have been lost" ) + elif attribute == "batch_state": + value = numpy.zeros((0,), numpy.uint8) else: - value = eval(value) + value = literal_eval(value) if attribute in skip_attributes: continue From 07c6ba1cd07449bf45f6dae7ee98df636fe4096e Mon Sep 17 00:00:00 2001 From: Beth Cimini Date: Tue, 20 Dec 2022 18:58:20 +0000 Subject: [PATCH 47/52] Allow notes to have pipes without breaking pipeline loading (#136) * split out note by regex rather than pipes * Add back support for older pipelines --- cellprofiler_core/pipeline/_pipeline.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index f5e742f8..20f709b7 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -603,7 +603,15 @@ def setup_module( raise ValueError("Invalid format for attributes: %s" % attribute_string) # Fix for array dtypes which contain split separator attribute_string = attribute_string.replace("dtype='|S", "dtype='S") - attribute_strings = attribute_string[1:-1].split("|") + if len(re.split("(?P\|notes:\[.*?\]\|)",attribute_string)) ==3: + # 4674- sometimes notes have pipes + prenote,note,postnote = re.split("(?P\|notes:\[.*?\]\|)",attribute_string) + attribute_strings = prenote[1:].split("|") + attribute_strings += [note[1:-1]] + attribute_strings += postnote[:-1].split("|") + else: + #old or weird pipeline without notes + attribute_strings = attribute_string[1:-1].split("|") variable_revision_number = None for a in attribute_strings: From 865728bd79f1a12e9d2b716e6532c8b6e1c2ba53 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Wed, 21 Dec 2022 15:40:57 -0500 Subject: [PATCH 48/52] Reset default metadata module state, on automatic extraction --- cellprofiler_core/pipeline/_pipeline.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index 20f709b7..2e02fb29 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -579,8 +579,15 @@ def setup_modules(self, fd, module_count, raise_on_error): new_modules.append(module) module_number += 1 if isinstance(module, Metadata) and module.removed_automatic_extraction: + # turn on extraction in Images, if metadata module was enabled if isinstance(new_modules[0], Images) and module.wants_metadata.value: new_modules[0].want_split.value = True + # now disable metdata module if the only extraction group was + # extracting from image file headers; and reset the default + # extraction method + if module.extraction_method_count.value == 0: + module.wants_metadata.value = 'No' + module.add_extraction_method(False) return new_modules def setup_module( From 52c0d2dc87906150283e594d62405f7f4403d527 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 22 Dec 2022 14:02:38 -0500 Subject: [PATCH 49/52] Always allow FileLocation as metadata key --- cellprofiler_core/constants/measurement.py | 2 +- cellprofiler_core/modules/metadata.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cellprofiler_core/constants/measurement.py b/cellprofiler_core/constants/measurement.py index e84b3221..4a399a94 100644 --- a/cellprofiler_core/constants/measurement.py +++ b/cellprofiler_core/constants/measurement.py @@ -69,7 +69,7 @@ C_OBJECTS_Z = "ObjectsZPlane" C_OBJECTS_T = "ObjectsTimepoint" C_CHANNEL_TYPE = "ChannelType" -C_FILE_LOCATION = "File_Location" +C_FILE_LOCATION = "FileLocation" M_METADATA_TAGS = "_".join((C_METADATA, "Tags")) M_GROUPING_TAGS = "_".join((C_METADATA, "GroupingTags")) RESERVED_METADATA_KEYS = (C_URL, C_SERIES, C_SERIES_NAME, C_FRAME, C_FILE_LOCATION, diff --git a/cellprofiler_core/modules/metadata.py b/cellprofiler_core/modules/metadata.py index dd7351eb..2f0d6f0c 100644 --- a/cellprofiler_core/modules/metadata.py +++ b/cellprofiler_core/modules/metadata.py @@ -23,6 +23,7 @@ from ..constants.modules.metadata import CSV_JOIN_NAME from ..constants.modules.metadata import IPD_JOIN_NAME from ..constants.modules.metadata import COL_PATH +from ..constants.modules.metadata import COL_URL from ..constants.modules.metadata import COL_SERIES from ..constants.modules.metadata import DTC_ALL from ..constants.modules.metadata import DTC_CHOOSE @@ -1064,8 +1065,9 @@ def validate_module(self, pipeline): def get_metadata_keys(self): """Return a collection of metadata keys to be associated with files""" + keys = set([COL_URL]) if not self.wants_metadata: - return [] + return keys keys = set(DEFAULT_METADATA_TAGS) for extract_group in self.extraction_methods: if extract_group.extraction_method == X_MANUAL_EXTRACTION: From ea0aa72e01352d96e639e496df62c63476dc68f0 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 22 Dec 2022 19:45:08 -0500 Subject: [PATCH 50/52] Go back to 'C' for channel --- cellprofiler_core/measurement/_measurements.py | 4 ++-- cellprofiler_core/modules/images.py | 4 ++-- cellprofiler_core/modules/loaddata.py | 11 ++++++++--- cellprofiler_core/modules/metadata.py | 4 ++-- cellprofiler_core/modules/namesandtypes.py | 4 ++-- cellprofiler_core/pipeline/_image_plane.py | 6 +++--- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/cellprofiler_core/measurement/_measurements.py b/cellprofiler_core/measurement/_measurements.py index 4f7256ad..c2431e92 100644 --- a/cellprofiler_core/measurement/_measurements.py +++ b/cellprofiler_core/measurement/_measurements.py @@ -20,7 +20,7 @@ from ..constants.measurement import COLTYPE_FLOAT from ..constants.measurement import COLTYPE_INTEGER from ..constants.measurement import COLTYPE_VARCHAR -from ..constants.measurement import C_CHANNEL +from ..constants.measurement import C_C from ..constants.measurement import C_CHANNEL_TYPE from ..constants.measurement import C_FILE_NAME from ..constants.measurement import C_FRAME @@ -1313,7 +1313,7 @@ def write_image_sets(self, fd_or_file, start=None, stop=None): C_FILE_NAME, C_SERIES, C_FRAME, - C_CHANNEL, + C_C, C_Z, C_T, C_OBJECTS_URL, diff --git a/cellprofiler_core/modules/images.py b/cellprofiler_core/modules/images.py index 18b8e5ff..08e4bfca 100644 --- a/cellprofiler_core/modules/images.py +++ b/cellprofiler_core/modules/images.py @@ -2,7 +2,7 @@ import itertools import logging -from cellprofiler_core.constants.measurement import C_SERIES_NAME, C_CHANNEL, C_SERIES, C_Z, C_T +from cellprofiler_core.constants.measurement import C_SERIES_NAME, C_C, C_SERIES, C_Z, C_T from cellprofiler_core.pipeline import ImagePlane from cellprofiler_core.constants.module import FILTER_RULES_BUTTONS_HELP from cellprofiler_core.constants.modules.images import FILTER_CHOICE_ALL @@ -353,7 +353,7 @@ def get_metadata_keys(self): return [] result = [C_SERIES, C_SERIES_NAME] if self.split_C.value: - result.append(C_CHANNEL) + result.append(C_C) if self.split_T.value: result.append(C_T) if self.split_Z.value: diff --git a/cellprofiler_core/modules/loaddata.py b/cellprofiler_core/modules/loaddata.py index 74428a00..943def88 100644 --- a/cellprofiler_core/modules/loaddata.py +++ b/cellprofiler_core/modules/loaddata.py @@ -9,7 +9,8 @@ from ..constants.image import C_MD5_DIGEST from ..constants.image import C_SCALING from ..constants.image import C_WIDTH -from ..constants.measurement import COLTYPE_FLOAT, C_CHANNEL, C_Z, C_T, C_OBJECTS_CHANNEL, C_OBJECTS_Z, C_OBJECTS_T +from ..constants.measurement import C_CHANNEL, C_C, C_Z, C_T, C_OBJECTS_CHANNEL, C_OBJECTS_Z, C_OBJECTS_T +from ..constants.measurement import COLTYPE_FLOAT from ..constants.measurement import COLTYPE_INTEGER from ..constants.measurement import COLTYPE_VARCHAR from ..constants.measurement import COLTYPE_VARCHAR_FILE_NAME @@ -1043,7 +1044,7 @@ def fetch_provider(self, name, measurements, is_image_name=True): url_feature = f"{C_URL}_{name}" series_feature = f"{C_SERIES}_{name}" frame_feature = f"{C_FRAME}_{name}" - channel_feature = f"{C_CHANNEL}_{name}" + channel_feature = f"{C_C}_{name}" plane_feature = f"{C_Z}_{name}" timepoint_feature = f"{C_T}_{name}" else: @@ -1059,7 +1060,11 @@ def fetch_provider(self, name, measurements, is_image_name=True): keyword_args = {} for feature in (series_feature, frame_feature, channel_feature, plane_feature, timepoint_feature): - if measurements.has_feature("Image", feature): + if measurements.has_feature("Image", feature) or ( + is_image_name + and feature == C_CHANNEL + and measurements.has_feature("Image", feature) + ): val = measurements["Image", feature] if numpy.isnan(val): val = None diff --git a/cellprofiler_core/modules/metadata.py b/cellprofiler_core/modules/metadata.py index 2f0d6f0c..389fa186 100644 --- a/cellprofiler_core/modules/metadata.py +++ b/cellprofiler_core/modules/metadata.py @@ -8,7 +8,7 @@ import urllib.request from ..constants.image import MD_SIZE_C, MD_SIZE_T, MD_SIZE_Z, MD_SIZE_X, MD_SIZE_Y -from ..constants.measurement import COLTYPE_FLOAT, C_Z, C_T, C_CHANNEL, FTR_WELL, ROW_KEYS, COL_KEYS +from ..constants.measurement import COLTYPE_FLOAT, C_Z, C_T, C_C, FTR_WELL, ROW_KEYS, COL_KEYS from ..constants.measurement import COLTYPE_INTEGER from ..constants.measurement import COLTYPE_VARCHAR from ..constants.measurement import COLTYPE_VARCHAR_FILE_NAME @@ -1097,7 +1097,7 @@ def get_dt_metadata_keys(self): MD_SIZE_Y, C_SERIES, C_FRAME, - C_CHANNEL, + C_C, C_T, C_Z, ) diff --git a/cellprofiler_core/modules/namesandtypes.py b/cellprofiler_core/modules/namesandtypes.py index 01891218..0fbdcbf8 100644 --- a/cellprofiler_core/modules/namesandtypes.py +++ b/cellprofiler_core/modules/namesandtypes.py @@ -25,7 +25,7 @@ from ..constants.measurement import COLTYPE_VARCHAR_FILE_NAME from ..constants.measurement import COLTYPE_VARCHAR_FORMAT from ..constants.measurement import COLTYPE_VARCHAR_PATH_NAME -from ..constants.measurement import C_CHANNEL +from ..constants.measurement import C_C from ..constants.measurement import C_COUNT from ..constants.measurement import C_FILE_NAME from ..constants.measurement import C_LOCATION @@ -1088,7 +1088,7 @@ def prepare_run(self, workspace): series_category = C_SERIES series_name_category = C_SERIES_NAME frame_category = C_FRAME - channel_category = C_CHANNEL + channel_category = C_C z_category = C_Z t_category = C_T diff --git a/cellprofiler_core/pipeline/_image_plane.py b/cellprofiler_core/pipeline/_image_plane.py index 0b8825a0..5b06be3f 100644 --- a/cellprofiler_core/pipeline/_image_plane.py +++ b/cellprofiler_core/pipeline/_image_plane.py @@ -3,7 +3,7 @@ from cellprofiler_core.constants.measurement import RESERVED_METADATA_KEYS, \ C_MONOCHROME, C_RGB, C_TILE, C_URL, \ - C_SERIES, C_CHANNEL, C_Z, C_T, C_INDEX, C_SERIES_NAME + C_SERIES, C_C, C_Z, C_T, C_INDEX, C_SERIES_NAME from cellprofiler_core.pipeline import ImageFile LOGGER = logging.getLogger(__name__) @@ -40,7 +40,7 @@ def __init__(self, file: ImageFile, series=None, index=None, channel=None, z=Non self._metadata_dict[C_SERIES] = series self._metadata_dict[C_SERIES_NAME] = name self._metadata_dict[C_INDEX] = index - self._metadata_dict[C_CHANNEL] = channel + self._metadata_dict[C_C] = channel self._metadata_dict[C_T] = t self._metadata_dict[C_Z] = z self._metadata_dict[C_TILE] = xywh @@ -115,7 +115,7 @@ def index(self): @property def channel(self): - return self._metadata_dict[C_CHANNEL] + return self._metadata_dict[C_C] @property def z(self): From eb50f3a007a52afd9d49fa9c66cfe40e58e92c55 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 5 Jan 2023 12:08:02 -0500 Subject: [PATCH 51/52] Remove deprecated numpy aliases (#140) * Remove deprecated numpy aliases * Recompile centrosome --- cellprofiler_core/measurement/_measurements.py | 2 +- cellprofiler_core/utilities/hdf5_dict.py | 4 ++-- tests/modules/test_loaddata.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cellprofiler_core/measurement/_measurements.py b/cellprofiler_core/measurement/_measurements.py index c2431e92..1d445ab2 100644 --- a/cellprofiler_core/measurement/_measurements.py +++ b/cellprofiler_core/measurement/_measurements.py @@ -224,7 +224,7 @@ def initialize(self, measurement_columns): def fix_type(t): if t == "integer": - return numpy.int + return int if t.startswith("varchar"): len = t.split("(")[1][:-1] return numpy.dtype("a" + len) diff --git a/cellprofiler_core/utilities/hdf5_dict.py b/cellprofiler_core/utilities/hdf5_dict.py index 0d519e9c..5bb6db59 100644 --- a/cellprofiler_core/utilities/hdf5_dict.py +++ b/cellprofiler_core/utilities/hdf5_dict.py @@ -1184,7 +1184,7 @@ def add_files_to_filelist(self, urls): marker = 'REPLACE' inserted = numpy.insert(existing_meta, insertion_indexes, marker).tolist() inserted = [x if isinstance(x, numpy.ndarray) else blank_metadata for x in inserted] - metadataset[:] = numpy.array(inserted, dtype=numpy.object) + metadataset[:] = numpy.array(inserted, dtype=object) # Now we insert the series name fields. seriesnameset[:] = numpy.insert(existing_names, insertion_indexes, blank_names) else: @@ -1214,7 +1214,7 @@ def add_files_to_filelist(self, urls): maxshape=(None, ) ) md[:] = numpy.array([blank_metadata] * len(filenames), - dtype=numpy.object) + dtype=object) self.hdf5_file.flush() self.notify() diff --git a/tests/modules/test_loaddata.py b/tests/modules/test_loaddata.py index c572f4c5..24f03129 100644 --- a/tests/modules/test_loaddata.py +++ b/tests/modules/test_loaddata.py @@ -244,7 +244,7 @@ def test_int_image_measurement(): pipeline, module, filename = make_pipeline(csv_text) m = pipeline.run() data = m.get_current_image_measurement("Test_Measurement") - assert isinstance(data, numpy.integer), "data is type %s, not np.int" % (type(data)) + assert isinstance(data, numpy.integer), "data is type %s, not int" % (type(data)) assert data == 1 os.remove(filename) From c37c17aa1777c2e83b98145c7d708f79dbf3c624 Mon Sep 17 00:00:00 2001 From: Nodar Gogoberidze Date: Thu, 5 Jan 2023 14:46:37 -0500 Subject: [PATCH 52/52] Comment out omero login handler --- cellprofiler_core/worker/_worker.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index 7db7768b..c3da62f7 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -357,13 +357,14 @@ def post_group_display_handler(self, module, display_data, image_set_number): ) rep = self.send(req) - def omero_login_handler(self): - """Handle requests for an Omero login""" - from cellprofiler_core.bioformats.formatreader import use_omero_credentials - - req = OmeroLogin(self.current_analysis_id) - rep = self.send(req) - use_omero_credentials(rep.credentials) + #TODO: disabled until CellProfiler/CellProfiler#4684 is resolved + # def omero_login_handler(self): + # """Handle requests for an Omero login""" + # from cellprofiler_core.bioformats.formatreader import use_omero_credentials + + # req = OmeroLogin(self.current_analysis_id) + # rep = self.send(req) + # use_omero_credentials(rep.credentials) def send(self, req, work_socket=None): """Send a request and receive a reply