diff --git a/cellprofiler_core/analysis/_runner.py b/cellprofiler_core/analysis/_runner.py index 9ae20b1d..8dbf450e 100644 --- a/cellprofiler_core/analysis/_runner.py +++ b/cellprofiler_core/analysis/_runner.py @@ -8,6 +8,7 @@ import tempfile import threading from typing import List, Any +import re import numpy import psutil @@ -21,7 +22,6 @@ from . import request as anarequest from ..image import ImageSetList from ..measurement import Measurements -from ..utilities.java import JAVA_STARTED from ..utilities.measurement import load_measurements_from_buffer from ..pipeline import dump from ..preferences import get_plugin_directory @@ -38,6 +38,8 @@ from ..workspace import Workspace +LOGGER = logging.getLogger(__name__) + class Runner: """The Runner manages two threads (per instance) and all of the workers (per class, i.e., singletons). @@ -167,22 +169,22 @@ def notify_threads(self): def cancel(self): """cancel the analysis run""" - logging.debug("Stopping workers") + LOGGER.debug("Stopping workers") self.stop_workers() - logging.debug("Canceling run") + LOGGER.debug("Canceling run") self.cancelled = True self.paused = False self.notify_threads() - logging.debug("Waiting on interface thread") + LOGGER.debug("Waiting on interface thread") self.interface_thread.join() - logging.debug("Waiting on jobserver thread") + LOGGER.debug("Waiting on jobserver thread") self.jobserver_thread.join() self.interface_thread = None self.jobserver_thread = None self.work_queue = queue.Queue() self.in_process_queue = queue.Queue() self.finished_queue = queue.Queue() - logging.debug("Cancel complete") + LOGGER.debug("Cancel complete") def pause(self): """pause the analysis run""" @@ -214,8 +216,6 @@ def interface( image_set_end - last image set number to process overwrite - whether to recompute imagesets that already have data in initial_measurements. """ - from javabridge import attach, detach - posted_analysis_started = False acknowledged_thread_start = False measurements = None @@ -426,9 +426,6 @@ def interface( except Exception as e: print(e) finally: - if JAVA_STARTED: - import javabridge - javabridge.detach() # Note - the measurements file is owned by the queue consumer # after this post_event. # @@ -529,21 +526,21 @@ def jobserver(self, start_signal): continue if isinstance(req, anarequest.PipelinePreferences): - logging.debug("Received pipeline preferences request") + LOGGER.debug("Received pipeline preferences request") req.reply( Reply( pipeline_blob=numpy.array(self.pipeline_as_string()), preferences=preferences_as_dict(), ) ) - logging.debug("Replied to pipeline preferences request") + LOGGER.debug("Replied to pipeline preferences request") elif isinstance(req, anarequest.InitialMeasurements): - logging.debug("Received initial measurements request") + LOGGER.debug("Received initial measurements request") req.reply(Reply(buf=self.initial_measurements_buf)) - logging.debug("Replied to initial measurements request") + LOGGER.debug("Replied to initial measurements request") elif isinstance(req, anarequest.Work): if not self.work_queue.empty(): - logging.debug("Received work request") + LOGGER.debug("Received work request") ( job, worker_runs_post_group, @@ -557,7 +554,7 @@ def jobserver(self, start_signal): ) ) self.queue_dispatched_job(job) - logging.debug( + LOGGER.debug( "Dispatched job: image sets=%s" % ",".join([str(i) for i in job]) ) @@ -568,21 +565,21 @@ def jobserver(self, start_signal): elif isinstance(req, anareply.ImageSetSuccess): # interface() is responsible for replying, to allow it to # request the shared_state dictionary if needed. - logging.debug("Received ImageSetSuccess") + LOGGER.debug("Received ImageSetSuccess") self.queue_imageset_finished(req) - logging.debug("Enqueued ImageSetSuccess") + LOGGER.debug("Enqueued ImageSetSuccess") elif isinstance(req, anarequest.SharedDictionary): - logging.debug("Received shared dictionary request") + LOGGER.debug("Received shared dictionary request") req.reply(anareply.SharedDictionary(dictionaries=self.shared_dicts)) - logging.debug("Sent shared dictionary reply") + LOGGER.debug("Sent shared dictionary reply") elif isinstance(req, anarequest.MeasurementsReport): - logging.debug("Received measurements report") + LOGGER.debug("Received measurements report") self.queue_received_measurements(req.image_set_numbers, req.buf) req.reply(anareply.Ack()) - logging.debug("Acknowledged measurements report") + LOGGER.debug("Acknowledged measurements report") elif isinstance(req, anarequest.AnalysisCancel): # Signal the interface that we are cancelling - logging.debug("Received analysis worker cancel request") + LOGGER.debug("Received analysis worker cancel request") with self.interface_work_cv: self.cancelled = True self.interface_work_cv.notify() @@ -599,13 +596,13 @@ def jobserver(self, start_signal): anarequest.OmeroLogin, ), ): - logging.debug("Enqueueing interactive request") + LOGGER.debug("Enqueueing interactive request") # bump upward self.post_event(req) - logging.debug("Interactive request enqueued") + LOGGER.debug("Interactive request enqueued") else: msg = "Unknown request from worker: %s of type %s" % (req, type(req)) - logging.error(msg) + LOGGER.error(msg) raise ValueError(msg) # stop the ZMQ-boundary thread - will also deal with any requests waiting on replies @@ -647,7 +644,7 @@ def start_workers(self, num=None): boundary = self.boundary - logging.info("Starting workers on address %s" % boundary.request_address) + LOGGER.info("Starting workers on address %s" % boundary.request_address) close_fds = False @@ -663,7 +660,9 @@ def start_workers(self, num=None): "--conserve-memory", str(get_conserve_memory()), "--always-continue", - str(get_always_continue()) + str(get_always_continue()), + "--log-level", + str(logging.root.level) ] # start workers @@ -707,8 +706,18 @@ def run_logger(workR, widx): line = line.decode("utf-8") if not line: break - logging.info("Worker %d: %s", widx, line.rstrip()) - except: + log_msg_match = re.match(fr"{workR.pid}\|(10|20|30|40|50)\|(.*)", line) + if log_msg_match: + levelno = int(log_msg_match.group(1)) + msg = log_msg_match.group(2) + else: + levelno = 20 + msg = line + + LOGGER.log(levelno, "\n\r [Worker %d (%d)] %s", widx, workR.pid, msg.rstrip()) + + except Exception as e: + LOGGER.exception(e) break start_daemon_thread( diff --git a/cellprofiler_core/bioformats/__init__.py b/cellprofiler_core/bioformats/__init__.py new file mode 100644 index 00000000..9f5cf72a --- /dev/null +++ b/cellprofiler_core/bioformats/__init__.py @@ -0,0 +1 @@ +# NB: No implementation needed. diff --git a/cellprofiler_core/bioformats/formatreader.py b/cellprofiler_core/bioformats/formatreader.py new file mode 100644 index 00000000..3b2e8b38 --- /dev/null +++ b/cellprofiler_core/bioformats/formatreader.py @@ -0,0 +1,19 @@ +#TODO: unimplemented until CellProfiler/CellProfiler#4684 is resolved + +K_OMERO_SERVER = None +K_OMERO_PORT = None +K_OMERO_USER = None +K_OMERO_SESSION_ID = None +K_OMERO_CONFIG_FILE = None + +def clear_image_reader_cache(): + raise RuntimeError("unimplemented") + +def set_omero_login_hook(omero_login): + raise RuntimeError("unimplemented") + +def get_omero_credentials(): + raise RuntimeError("unimplemented") + +def use_omero_credentials(credentials): + raise RuntimeError("unimplemented") diff --git a/cellprofiler_core/bioformats/formatwriter.py b/cellprofiler_core/bioformats/formatwriter.py new file mode 100644 index 00000000..99f3f123 --- /dev/null +++ b/cellprofiler_core/bioformats/formatwriter.py @@ -0,0 +1,104 @@ +import os +import scyjava +import numpy as np +import logging + +from . import omexml +from ..utilities.java import jimport + + +LOGGER = logging.getLogger(__name__) + +p2j = lambda v: scyjava.to_java(v) + +def write_image( + pathname, + pixels, + pixel_type, + c, + z, + t, + size_c, + size_z, + size_t, + channel_names +): + # https://www.javadoc.io/doc/org.openmicroscopy/ome-common/5.3.2/loci/common/services/ServiceFactory.html + OMEXMLServiceFactory = jimport("loci.common.services.ServiceFactory") + # https://javadoc.scijava.org/Bio-Formats/loci/formats/services/OMEXMLService.html + OMEXMLService = jimport("loci.formats.services.OMEXMLService") + # https://javadoc.scijava.org/Bio-Formats/loci/formats/ImageWriter.html + ImageWriter = jimport("loci.formats.ImageWriter") + # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/IMetadata.html + IMetadata = jimport("loci.formats.meta.IMetadata") + + DimensionsOrder = jimport("ome.xml.model.enums.DimensionOrder") + PixelType = jimport("ome.xml.model.enums.PixelType") + PositiveInteger = jimport("ome.xml.model.primitives.PositiveInteger") + + omexml_service = OMEXMLServiceFactory().getInstance(OMEXMLService) + # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/OMEXMLMetadata.html + # https://www.javadoc.io/static/org.openmicroscopy/ome-xml/6.3.1/ome/xml/meta/MetadataStore.html + metadata = omexml_service.createOMEXMLMetadata() + metadata.createRoot() + + metadata.setImageName(os.path.split(pathname)[1], 0) + metadata.setPixelsSizeX(PositiveInteger(p2j(pixels.shape[1])), 0) + metadata.setPixelsSizeY(PositiveInteger(p2j(pixels.shape[0])), 0) + metadata.setPixelsSizeC(PositiveInteger(p2j(size_c)), 0) + metadata.setPixelsSizeZ(PositiveInteger(p2j(size_z)), 0) + metadata.setPixelsSizeT(PositiveInteger(p2j(size_t)), 0) + metadata.setPixelsBinDataBigEndian(True, 0, 0) + metadata.setPixelsDimensionOrder(DimensionsOrder.XYCTZ, 0) + metadata.setPixelsType(PixelType.fromString(pixel_type), 0) + + + if pixels.ndim == 3: + metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0) + metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(pixels.shape[2])), 0, 0) + omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2])) + # omexml.structured_annotations.add_original_metadata( + # ome.OM_SAMPLES_PER_PIXEL, str(pixels.shape[2])) + elif size_c > 1: + # meta.channel_count = size_c <- cant find + metadata.setPixelsSizeC(PositiveInteger(p2j(pixels.shape[2])), 0) + omexml_service.populateOriginalMetadata(metadata, "SamplesPerPixel", str(pixels.shape[2])) + + metadata.setImageID("Image:0", 0) + metadata.setPixelsID("Pixels:0", 0) + + for i in range(size_c): + metadata.setChannelID(f"Channel:0:{i}", 0, i) + metadata.setChannelSamplesPerPixel(PositiveInteger(p2j(1)), 0, i) + + index = c + size_c * z + size_c * size_z * t + pixel_buffer = convert_pixels_to_buffer(pixels, pixel_type) + + + writer = ImageWriter() + writer.setMetadataRetrieve(metadata) + writer.setId(pathname) + writer.setInterleaved(True) + writer.saveBytes(index, pixel_buffer) + writer.close() + +def convert_pixels_to_buffer(pixels, pixel_type): + '''Convert the pixels in the image into a buffer of the right pixel type + + pixels - a 2d monochrome or color image + + pixel_type - one of the OME pixel types + + returns a 1-d byte array + ''' + if pixel_type == omexml.PT_UINT8: + as_dtype = np.uint8 + elif pixel_type == omexml.PT_UINT16: + as_dtype = " max_value: + if value and value > max_value: max_value = value result += str(max_value) else: @@ -1273,7 +1271,7 @@ def load_image_sets(self, fd_or_file, start=None, stop=None): column.append(field) last_image_number = image_number if last_image_number == 0: - logging.warn("No image sets were loaded") + LOGGER.warn("No image sets were loaded") return if start is None: image_numbers = list(range(1, last_image_number + 1)) @@ -1315,7 +1313,7 @@ def write_image_sets(self, fd_or_file, start=None, stop=None): C_FILE_NAME, C_SERIES, C_FRAME, - C_CHANNEL, + C_C, C_Z, C_T, C_OBJECTS_URL, @@ -1606,7 +1604,7 @@ def get_image( ) if image.pixel_data.shape[-1] == 4: - logging.warning("Discarding alpha channel.") + LOGGER.warning("Discarding alpha channel.") return RGBImage(image) diff --git a/cellprofiler_core/module/_module.py b/cellprofiler_core/module/_module.py index 462bd298..0403e056 100644 --- a/cellprofiler_core/module/_module.py +++ b/cellprofiler_core/module/_module.py @@ -21,6 +21,8 @@ from ..setting import ValidationError +LOGGER = logging.getLogger(__name__) + class Module: """ Derive from the abstract module class to create your own module in Python @@ -111,7 +113,7 @@ def from_dict(self, settings: list, attributes: dict): self.module_name = attributes["module_name"] setting_values = [setting["value"] for setting in settings] if attributes["variable_revision_number"] > self.variable_revision_number: - logging.warning(f"Loaded module '{self.module_name}' is from a newer version of CellProfiler - " + LOGGER.warning(f"Loaded module '{self.module_name}' is from a newer version of CellProfiler - " f"v{attributes['variable_revision_number']}, current version is " f"v{self.variable_revision_number}. Settings may load incorrectly.") self.set_settings_from_values( @@ -398,8 +400,8 @@ def test_valid(self, pipeline): for setting in self.visible_settings(): setting.test_valid(pipeline) self.validate_module(pipeline) - except ValidationError as instance: - raise instance + except ValidationError as ve: + raise ve except Exception as e: raise ValidationError( "Exception in cpmodule.test_valid %s" % e, self.visible_settings()[0] diff --git a/cellprofiler_core/modules/images.py b/cellprofiler_core/modules/images.py index 24304d21..08e4bfca 100644 --- a/cellprofiler_core/modules/images.py +++ b/cellprofiler_core/modules/images.py @@ -2,7 +2,7 @@ import itertools import logging -from cellprofiler_core.constants.measurement import C_SERIES_NAME, C_CHANNEL, C_SERIES, C_Z, C_T +from cellprofiler_core.constants.measurement import C_SERIES_NAME, C_C, C_SERIES, C_Z, C_T from cellprofiler_core.pipeline import ImagePlane from cellprofiler_core.constants.module import FILTER_RULES_BUTTONS_HELP from cellprofiler_core.constants.modules.images import FILTER_CHOICE_ALL @@ -24,6 +24,9 @@ from cellprofiler_core.utilities.image import image_resource from cellprofiler_core.utilities.image import is_image + +LOGGER = logging.getLogger(__name__) + __doc__ = """\ Images ====== @@ -290,7 +293,7 @@ def on_activated(self, workspace): def filter_file_list(self, workspace): file_list = workspace.pipeline.file_list if file_list and not workspace.file_list.has_files(): - logging.warning("Workspace file list is empty, will populate from pipeline." + LOGGER.warning("Workspace file list is empty, will populate from pipeline." "This may happen if you're running in headless mode.") workspace.file_list.add_files_to_filelist([f.url for f in file_list]) if self.filter_choice != FILTER_CHOICE_NONE: @@ -314,7 +317,7 @@ def prepare_run(self, workspace): return True file_list = self.filter_file_list(workspace) if self.want_split.value: - logging.debug("Metadata extraction will be performed now if needed") + LOGGER.debug("Metadata extraction will be performed now if needed") if self.extract_metadata.callback is not None: # If GUI is present perform extraction and refresh the file list GUI self.extract_metadata.callback() @@ -350,7 +353,7 @@ def get_metadata_keys(self): return [] result = [C_SERIES, C_SERIES_NAME] if self.split_C.value: - result.append(C_CHANNEL) + result.append(C_C) if self.split_T.value: result.append(C_T) if self.split_Z.value: diff --git a/cellprofiler_core/modules/loaddata.py b/cellprofiler_core/modules/loaddata.py index 74428a00..943def88 100644 --- a/cellprofiler_core/modules/loaddata.py +++ b/cellprofiler_core/modules/loaddata.py @@ -9,7 +9,8 @@ from ..constants.image import C_MD5_DIGEST from ..constants.image import C_SCALING from ..constants.image import C_WIDTH -from ..constants.measurement import COLTYPE_FLOAT, C_CHANNEL, C_Z, C_T, C_OBJECTS_CHANNEL, C_OBJECTS_Z, C_OBJECTS_T +from ..constants.measurement import C_CHANNEL, C_C, C_Z, C_T, C_OBJECTS_CHANNEL, C_OBJECTS_Z, C_OBJECTS_T +from ..constants.measurement import COLTYPE_FLOAT from ..constants.measurement import COLTYPE_INTEGER from ..constants.measurement import COLTYPE_VARCHAR from ..constants.measurement import COLTYPE_VARCHAR_FILE_NAME @@ -1043,7 +1044,7 @@ def fetch_provider(self, name, measurements, is_image_name=True): url_feature = f"{C_URL}_{name}" series_feature = f"{C_SERIES}_{name}" frame_feature = f"{C_FRAME}_{name}" - channel_feature = f"{C_CHANNEL}_{name}" + channel_feature = f"{C_C}_{name}" plane_feature = f"{C_Z}_{name}" timepoint_feature = f"{C_T}_{name}" else: @@ -1059,7 +1060,11 @@ def fetch_provider(self, name, measurements, is_image_name=True): keyword_args = {} for feature in (series_feature, frame_feature, channel_feature, plane_feature, timepoint_feature): - if measurements.has_feature("Image", feature): + if measurements.has_feature("Image", feature) or ( + is_image_name + and feature == C_CHANNEL + and measurements.has_feature("Image", feature) + ): val = measurements["Image", feature] if numpy.isnan(val): val = None diff --git a/cellprofiler_core/modules/metadata.py b/cellprofiler_core/modules/metadata.py index ac5327c0..5936141e 100644 --- a/cellprofiler_core/modules/metadata.py +++ b/cellprofiler_core/modules/metadata.py @@ -8,7 +8,7 @@ import urllib.request from ..constants.image import MD_SIZE_C, MD_SIZE_T, MD_SIZE_Z, MD_SIZE_X, MD_SIZE_Y -from ..constants.measurement import COLTYPE_FLOAT, C_Z, C_T, C_CHANNEL, FTR_WELL, ROW_KEYS, COL_KEYS +from ..constants.measurement import COLTYPE_FLOAT, C_Z, C_T, C_C, FTR_WELL, ROW_KEYS, COL_KEYS from ..constants.measurement import COLTYPE_INTEGER from ..constants.measurement import COLTYPE_VARCHAR from ..constants.measurement import COLTYPE_VARCHAR_FILE_NAME @@ -23,6 +23,7 @@ from ..constants.modules.metadata import CSV_JOIN_NAME from ..constants.modules.metadata import IPD_JOIN_NAME from ..constants.modules.metadata import COL_PATH +from ..constants.modules.metadata import COL_URL from ..constants.modules.metadata import COL_SERIES from ..constants.modules.metadata import DTC_ALL from ..constants.modules.metadata import DTC_CHOOSE @@ -73,6 +74,9 @@ from cellprofiler_core.utilities.image import image_resource from cellprofiler_core.utilities.measurement import find_metadata_tokens + +LOGGER = logging.getLogger(__name__) + __doc__ = """\ Metadata ======== @@ -695,7 +699,7 @@ def get_csv_header(self, group): group.imported_metadata_col_names = rdr.fieldnames fd.close() except Exception as e: - print("Error while decoding CSV:", e) + LOGGER.error(f"Error while decoding CSV - {csv_path}: {e.strerror}") return None return group.imported_metadata_col_names @@ -783,7 +787,7 @@ def visible_settings(self): result += [self.add_extraction_method_button] try: has_keys = len(self.get_dt_metadata_keys()) > 0 - except Exception: + except Exception as e: has_keys = False if has_keys: result += [self.dtc_divider, self.data_type_choice] @@ -888,7 +892,8 @@ def run_extraction(self, file_objects): file_object.add_metadata(candidate_dict) break if not valid: - print(f"No matching metadata found for {file_object.filename}") + LOGGER.info(f"No matching metadata found for {file_object.filename}") + break pass else: raise NotImplementedError(f"Invalid extraction method '{group.extraction_method}'") @@ -1060,15 +1065,16 @@ def validate_module(self, pipeline): def get_metadata_keys(self): """Return a collection of metadata keys to be associated with files""" + keys = set([COL_URL]) if not self.wants_metadata: - return [] + return keys keys = set(DEFAULT_METADATA_TAGS) for extract_group in self.extraction_methods: if extract_group.extraction_method == X_MANUAL_EXTRACTION: self.compile_regex(extract_group) keys.update(extract_group.regex_pattern.groupindex.keys()) elif extract_group.extraction_method == X_IMPORTED_EXTRACTION: - keys.update(self.get_csv_header(extract_group)) + keys.update(self.get_csv_header(extract_group) or '') if FTR_WELL not in keys and any(k in keys for k in ROW_KEYS) and any(k in keys for k in COL_KEYS): keys.add(FTR_WELL) return keys @@ -1079,7 +1085,7 @@ def get_dt_metadata_keys(self): """ return list( filter( - (lambda k: k not in self.NUMERIC_DATA_TYPES), self.get_metadata_keys() + (lambda k: k not in self.NUMERIC_DATA_TYPES), self.get_metadata_keys() or [] ) ) @@ -1091,7 +1097,7 @@ def get_dt_metadata_keys(self): MD_SIZE_Y, C_SERIES, C_FRAME, - C_CHANNEL, + C_C, C_T, C_Z, ) @@ -1130,13 +1136,13 @@ def apply_data_type(value, data_type): try: return int(value) except ValueError: - logging.warning(f"Metadata value {value} cannot be interpreted as an integer number.") + LOGGER.warning(f"Metadata value {value} cannot be interpreted as an integer number.") return value elif data_type == DataTypes.DT_FLOAT: try: return float(value) except ValueError: - logging.warning(f"Metadata value {value} cannot be interpreted as a floating point number.") + LOGGER.warning(f"Metadata value {value} cannot be interpreted as a floating point number.") return value return value diff --git a/cellprofiler_core/modules/namesandtypes.py b/cellprofiler_core/modules/namesandtypes.py index 0e11f52b..ec48c92d 100644 --- a/cellprofiler_core/modules/namesandtypes.py +++ b/cellprofiler_core/modules/namesandtypes.py @@ -25,7 +25,7 @@ from ..constants.measurement import COLTYPE_VARCHAR_FILE_NAME from ..constants.measurement import COLTYPE_VARCHAR_FORMAT from ..constants.measurement import COLTYPE_VARCHAR_PATH_NAME -from ..constants.measurement import C_CHANNEL +from ..constants.measurement import C_C from ..constants.measurement import C_COUNT from ..constants.measurement import C_FILE_NAME from ..constants.measurement import C_LOCATION @@ -89,6 +89,9 @@ ) from ..utilities.image import image_resource + +LOGGER = logging.getLogger(__name__) + __doc__ = """\ NamesAndTypes ============= @@ -1085,7 +1088,7 @@ def prepare_run(self, workspace): series_category = C_SERIES series_name_category = C_SERIES_NAME frame_category = C_FRAME - channel_category = C_CHANNEL + channel_category = C_C z_category = C_Z t_category = C_T @@ -1255,7 +1258,7 @@ def make_image_sets_by_order(self, image_planes): break errors = [] if not groups: - logging.warning("No images passed group filters") + LOGGER.warning("No images passed group filters") return [] desired_length = max([len(grp) for grp in groups.values()]) for name in self.get_column_names(want_singles=False): @@ -1375,7 +1378,7 @@ def handle_error_messages(errors): text = f"Metadata {error_info} for channel {error_chan} had {error_type}" else: text = f"Channel {error_chan} had {error_type}" - logging.warning(text) + LOGGER.warning(text) error_types[(error_type, error_chan)] += 1 if not get_headless(): msg = f"Warning: found {len(errors)} image set errors (see log for details)\n \n" diff --git a/cellprofiler_core/pipeline/_image_file.py b/cellprofiler_core/pipeline/_image_file.py index d8656c12..e533d926 100644 --- a/cellprofiler_core/pipeline/_image_file.py +++ b/cellprofiler_core/pipeline/_image_file.py @@ -13,7 +13,7 @@ from cellprofiler_core.reader import get_image_reader, Reader from cellprofiler_core.utilities.image import url_to_modpath, is_file_url -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class ImageFile: @@ -85,12 +85,12 @@ def extract_planes(self, workspace=None): try: reader = self.get_reader() except: - logger.error(f"May not be an image: {self.url}", exc_info=True) + LOGGER.error(f"May not be an image: {self.url}", exc_info=True) self.metadata[MD_SIZE_S] = 0 return meta_dict = reader.get_series_metadata() if meta_dict[MD_SIZE_S] == 0: - logger.error(f"File {self.filename} appears to contain no images.") + LOGGER.error(f"File {self.filename} appears to contain no images.") self.metadata[MD_SIZE_S] = 0 self.release_reader() return @@ -118,7 +118,7 @@ def load_plane_metadata(self, data, names=''): if len(data) < 5 or len(data) % 5 != 0: # No metadata or bad format if len(data) > 0: - logger.warning(f"Unable to load saved metadata for {self.filename}") + LOGGER.warning(f"Unable to load saved metadata for {self.filename}") return if len(data) == 5 and numpy.all(data == -1): # Unfilled metadata @@ -221,7 +221,7 @@ def add_metadata(self, meta_dict): # Used to bulk-add metadata keys from the Metadata module. Other modules should use set_metadata method. for key, value in meta_dict.items(): if key in RESERVED_METADATA_KEYS: - logger.error(f"Unable to set protected metadata key '{key}' to '{value}' for file {self.filename}. " + LOGGER.error(f"Unable to set protected metadata key '{key}' to '{value}' for file {self.filename}. " f"Please choose another key name.") continue self._metadata_dict[key] = value @@ -231,7 +231,7 @@ def set_metadata(self, key, value, force=False): raise PermissionError(f"Cannot override protected metadata key '{key}'") else: if force: - logger.warning(f"Overwriting protected key {key}. This may break functionality.") + LOGGER.warning(f"Overwriting protected key {key}. This may break functionality.") self._metadata_dict[key] = value def clear_metadata(self): diff --git a/cellprofiler_core/pipeline/_image_plane.py b/cellprofiler_core/pipeline/_image_plane.py index cb2b93b3..a76f73a6 100644 --- a/cellprofiler_core/pipeline/_image_plane.py +++ b/cellprofiler_core/pipeline/_image_plane.py @@ -3,10 +3,10 @@ from cellprofiler_core.constants.measurement import RESERVED_METADATA_KEYS, \ C_MONOCHROME, C_RGB, C_TILE, C_URL, \ - C_SERIES, C_CHANNEL, C_Z, C_T, C_INDEX, C_SERIES_NAME + C_SERIES, C_C, C_Z, C_T, C_INDEX, C_SERIES_NAME from cellprofiler_core.pipeline import ImageFile -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class ImagePlane: @@ -40,7 +40,7 @@ def __init__(self, file: ImageFile, series=None, index=None, channel=None, z=Non self._metadata_dict[C_SERIES] = series self._metadata_dict[C_SERIES_NAME] = name self._metadata_dict[C_INDEX] = index - self._metadata_dict[C_CHANNEL] = channel + self._metadata_dict[C_C] = channel self._metadata_dict[C_T] = t self._metadata_dict[C_Z] = z self._metadata_dict[C_TILE] = xywh @@ -115,7 +115,7 @@ def index(self): @property def channel(self): - return self.get_metadata(C_CHANNEL) + return self.get_metadata(C_C) @property def z(self): @@ -166,7 +166,7 @@ def set_metadata(self, key, value, force=False): raise PermissionError(f"Cannot override protected metadata key '{key}'") else: if force: - logger.warning(f"Overwriting protected key {key}. This may break functionality.") + LOGGER.warning(f"Overwriting protected key {key}. This may break functionality.") self._metadata_dict[key] = value diff --git a/cellprofiler_core/pipeline/_pipeline.py b/cellprofiler_core/pipeline/_pipeline.py index 4363ac04..2e02fb29 100644 --- a/cellprofiler_core/pipeline/_pipeline.py +++ b/cellprofiler_core/pipeline/_pipeline.py @@ -15,6 +15,7 @@ import urllib.request import uuid import weakref +from ast import literal_eval import numpy @@ -84,6 +85,7 @@ from ..constants.workspace import DISPOSITION_CANCEL from ..constants.workspace import DISPOSITION_PAUSE from ..constants.workspace import DISPOSITION_SKIP +from ..constants.image import PASSTHROUGH_SCHEMES from ..constants.modules.metadata import X_AUTOMATIC_EXTRACTION from ..image import ImageSetList from ..measurement import Measurements @@ -99,6 +101,9 @@ from cellprofiler_core import __version__ +LOGGER = logging.getLogger(__name__) + + def _is_fp(x): return hasattr(x, "seek") and hasattr(x, "read") @@ -234,7 +239,7 @@ def reload_modules(self): self.loadtxt(fd, raise_on_error=True) return True except Exception as e: - logging.warning( + LOGGER.warning( "Modules reloaded, but could not reinstantiate pipeline with new versions.", exc_info=True, ) @@ -258,7 +263,7 @@ def is_pipeline_txt_fd(fd): if header.startswith("CellProfiler Pipeline: http://www.cellprofiler.org"): return True if re.search(SAD_PROOFPOINT_COOKIE, header): - logging.info('print_emoji(":cat_crying_because_of_proofpoint:")') + LOGGER.info('print_emoji(":cat_crying_because_of_proofpoint:")') return True return False @@ -347,7 +352,7 @@ def load(self, fd_or_filename): @staticmethod def respond_to_version_mismatch_error(message): - logging.warning(message) + LOGGER.warning(message) def loadtxt(self, fp_or_filename, raise_on_error=False): """Load a pipeline from a text file @@ -398,8 +403,6 @@ def parse_pipeline(self, fd, raise_on_error, count=sys.maxsize): def validate_pipeline_file(self, fd, module_count): version = NATIVE_VERSION - do_deprecated_utf16_decode = False - do_utf16_decode = False has_image_plane_details = False git_hash = None pipeline_version = __version__ @@ -426,12 +429,6 @@ def validate_pipeline_file(self, fd, module_count): version, NATIVE_VERSION ) ) - elif 1 < version < 4: - do_deprecated_utf16_decode = True - elif version == 4: - do_utf16_decode = True - elif version == 5: - pass elif kwd in (H_SVN_REVISION, H_DATE_REVISION,): pipeline_version = int(value) elif kwd == H_MODULE_COUNT: @@ -453,6 +450,8 @@ def validate_pipeline_file(self, fd, module_count): return git_hash, has_image_plane_details, module_count, pipeline_version def validate_pipeline_version(self, current_version, git_hash, pipeline_version): + # if "pipeline_version" is an actual dave revision, ie unicode time + # at some point this changed to CellProfiler version (e.g. CP 4.2.4 would be 400) if 20080101000000 < pipeline_version < 30080101000000: # being optomistic... a millenium should be OK, no? second, minute, hour, day, month = [ @@ -474,18 +473,18 @@ def validate_pipeline_version(self, current_version, git_hash, pipeline_version) if (not get_headless()) and pipeline_version < current_version: if git_hash is not None: message = ( - "Your pipeline was saved using an old version\n" - "of CellProfiler (rev {}{}).\n" - "The current version of CellProfiler can load\n" - "and run this pipeline, but if you make changes\n" - "to it and save, the older version of CellProfiler\n" - "(perhaps the version your collaborator has?) may\n" - "not be able to load it.\n\n" - "You can ignore this warning if you do not plan to save\n" - "this pipeline or if you will only use it with this or\n" - "later versions of CellProfiler." + "\n\tYour pipeline was saved using an old version\n" + "\tof CellProfiler (rev {}{}).\n" + "\tThe current version of CellProfiler can load\n" + "\tand run this pipeline, but if you make changes\n" + "\tto it and save, the older version of CellProfiler\n" + "\t(perhaps the version your collaborator has?) may\n" + "\tnot be able to load it.\n\n" + "\tYou can ignore this warning if you do not plan to save\n" + "\tthis pipeline or if you will only use it with this or\n" + "\tlater versions of CellProfiler." ).format(git_hash, pipeline_date) - logging.warning(message) + LOGGER.warning(message) else: # pipeline versions pre-3.0.0 have unpredictable formatting if pipeline_version == 300: @@ -506,7 +505,7 @@ def validate_pipeline_version(self, current_version, git_hash, pipeline_version) "this pipeline or if you will only use it with this or\n" "later versions of CellProfiler." ).format(pipeline_version) - logging.warning(message) + LOGGER.warning(message) return pipeline_version @@ -571,7 +570,7 @@ def setup_modules(self, fd, module_count, raise_on_error): except Exception as instance: if raise_on_error: raise - logging.error("Failed to load pipeline", exc_info=True) + LOGGER.error("Failed to load pipeline", exc_info=True) event = LoadException(instance, module, module_name, settings) self.notify_listeners(event) if event.cancel_run: @@ -580,8 +579,15 @@ def setup_modules(self, fd, module_count, raise_on_error): new_modules.append(module) module_number += 1 if isinstance(module, Metadata) and module.removed_automatic_extraction: + # turn on extraction in Images, if metadata module was enabled if isinstance(new_modules[0], Images) and module.wants_metadata.value: new_modules[0].want_split.value = True + # now disable metdata module if the only extraction group was + # extracting from image file headers; and reset the default + # extraction method + if module.extraction_method_count.value == 0: + module.wants_metadata.value = 'No' + module.add_extraction_method(False) return new_modules def setup_module( @@ -604,13 +610,17 @@ def setup_module( raise ValueError("Invalid format for attributes: %s" % attribute_string) # Fix for array dtypes which contain split separator attribute_string = attribute_string.replace("dtype='|S", "dtype='S") - attribute_strings = attribute_string[1:-1].split("|") + if len(re.split("(?P\|notes:\[.*?\]\|)",attribute_string)) ==3: + # 4674- sometimes notes have pipes + prenote,note,postnote = re.split("(?P\|notes:\[.*?\]\|)",attribute_string) + attribute_strings = prenote[1:].split("|") + attribute_strings += [note[1:-1]] + attribute_strings += postnote[:-1].split("|") + else: + #old or weird pipeline without notes + attribute_strings = attribute_string[1:-1].split("|") variable_revision_number = None - # make batch_state decodable from text pipelines - # NOTE, MAGIC HERE: These variables are **necessary**, even though they - # aren't used anywhere obvious. Removing them **will** break these unit tests. - array = numpy.array - uint8 = numpy.uint8 + for a in attribute_strings: if a.isascii(): a = a.encode("utf-8").decode("unicode_escape") @@ -622,8 +632,8 @@ def setup_module( attribute, value = a.split(":", 1) if attribute == "notes": try: - value = eval(value) - except SyntaxError: + value = literal_eval(value) + except SyntaxError as e: value = value[1:-1].replace('"', "").replace("'", "").split(",") if any([chr(226) in line for line in value]): # There were some unusual UTF-8 characters present, let's try to fix them. @@ -633,16 +643,16 @@ def setup_module( for line in value ] except Exception as e: - print( - f"Error during notes decoding - {e} \nSome characters may have been lost" + LOGGER.error( + f"Error during notes decoding\n\t{e}\n\tSome characters may have been lost" ) + elif attribute == "batch_state": + value = numpy.zeros((0,), numpy.uint8) else: - value = eval(value) + value = literal_eval(value) if attribute in skip_attributes: continue - # En/decode needed to read example cppipe format - # TODO: remove en/decode when example cppipe no longer has \x__ characters - # value = eval(value.encode().decode("unicode_escape")) + if attribute == "variable_revision_number": variable_revision_number = value else: @@ -924,7 +934,7 @@ def run_with_yield( last_image_number = None - logging.info("Times reported are CPU and Wall-clock times for each module") + LOGGER.info("Times reported are CPU and Wall-clock times for each module") __group = self.group( grouping, @@ -1032,7 +1042,7 @@ def run_with_yield( self.run_module(module, workspace) except Exception as instance: print("pipeline_exception") - logging.error( + LOGGER.error( "Error detected during run of module %s", module.module_name, exc_info=True, @@ -1051,7 +1061,7 @@ def run_with_yield( cpu_delta_sec = max(0, cpu_t1 - cpu_t0) wall_delta_sec = max(0, wall_t1 - wall_t0) - logging.info( + LOGGER.info( "%s: Image # %d, module %s # %d: CPU_time = %.2f secs, Wall_time = %.2f secs" % ( start_time.ctime(), @@ -1071,7 +1081,7 @@ def run_with_yield( fig.Refresh() except Exception as instance: - logging.error( + LOGGER.error( "Failed to display results for module %s", module.module_name, exc_info=True, @@ -1230,18 +1240,18 @@ def run_image_set( if module in self.redundancy_map and len(self.modules()) > module.module_num: to_forget = self.redundancy_map[module] for image_name in to_forget: - logging.info(f"Releasing memory for redundant image {image_name}") + LOGGER.info(f"Releasing memory for redundant image {image_name}") workspace.image_set.clear_image(image_name) gc.collect() except Exception as e: - logging.warning(f"Encountered error during memory cleanup: {e}") + LOGGER.warning(f"Encountered error during memory cleanup: {e}") except CancelledException: # Analysis worker interaction handler is telling us that # the UI has cancelled the run. Forward exception upward. raise except Exception as exception: print("run_image_set_exception, get_always_continue",get_always_continue()) - logging.error( + LOGGER.error( "Error detected during run of module %s#%d", module.module_name, module.module_num, @@ -1265,7 +1275,7 @@ def run_image_set( cpu_t1 = sum(os_times[:-1]) cpu_delta_secs = max(0, cpu_t1 - cpu_t0) wall_delta_secs = max(0, wall_t1 - wall_t0) - logging.info( + LOGGER.info( "%s: Image # %d, module %s # %d: CPU_time = %.2f secs, Wall_time = %.2f secs" % ( start_time.ctime(), @@ -1493,7 +1503,7 @@ def on_pipeline_event(pipeline, event): self.clear_measurements(workspace.measurements) break except Exception as instance: - logging.error( + LOGGER.error( "Failed to prepare run for module %s", module.module_name, exc_info=True, @@ -1596,7 +1606,7 @@ def post_run(self, *args): try: module.post_run(workspace) except Exception as instance: - logging.error( + LOGGER.error( "Failed to complete post_run processing for module %s." % module.module_name, exc_info=True, @@ -1613,7 +1623,7 @@ def post_run(self, *args): workspace.post_run_display(module) except Exception as instance: # Warn about display failure but keep going. - logging.warning( + LOGGER.warning( "Caught exception during post_run_display for module %s." % module.module_name, exc_info=True, @@ -1645,7 +1655,7 @@ def prepare_to_create_batch(self, workspace, fn_alter_path): workspace.set_module(module) module.prepare_to_create_batch(workspace, fn_alter_path) except Exception as instance: - logging.error( + LOGGER.error( "Failed to collect batch information for module %s", module.module_name, exc_info=True, @@ -1741,7 +1751,7 @@ def prepare_group(self, workspace, grouping, image_numbers): try: module.prepare_group(workspace, grouping, image_numbers) except Exception as instance: - logging.error( + LOGGER.error( "Failed to prepare group in module %s", module.module_name, exc_info=True, @@ -1763,7 +1773,7 @@ def post_group(self, workspace, grouping): try: module.post_group(workspace, grouping) except Exception as instance: - logging.error( + LOGGER.error( "Failed during post-group processing for module %s" % module.module_name, exc_info=True, @@ -1779,7 +1789,7 @@ def post_group(self, workspace, grouping): try: workspace.post_group_display(module) except: - logging.warning( + LOGGER.warning( "Failed during post group display for module %s" % module.module_name, exc_info=True, @@ -1891,7 +1901,7 @@ def undo(): def enable_module(self, module): """Enable a module = make it executable""" if module.enabled: - logging.warning( + LOGGER.warning( "Asked to enable module %s, but it was already enabled" % module.module_name ) @@ -1908,7 +1918,7 @@ def undo(): def disable_module(self, module): """Disable a module = prevent it from being executed""" if not module.enabled: - logging.warning( + LOGGER.warning( "Asked to disable module %s, but it was already disabled" % module.module_name ) @@ -2036,7 +2046,7 @@ def load_file_list(self, workspace): try: urls, metadata, series_names = file_list.get_filelist(want_metadata=True) except Exception as instance: - logging.error("Failed to get file list from workspace", exc_info=True) + LOGGER.error("Failed to get file list from workspace", exc_info=True) x = IPDLoadException("Failed to get file list from workspace") self.notify_listeners(x) if x.cancel_run: @@ -2068,7 +2078,7 @@ def read_file_list(self, path_or_fd, add_undo=True): self.read_file_list(fd, add_undo=add_undo) elif any( pathname.startswith(protocol) - for protocol in ("http", "https", "ftp", "omero", "s3", "gs") + for protocol in PASSTHROUGH_SCHEMES ): with urllib.request.urlopen(pathname) as response: data = response.read().decode("utf-8").splitlines() diff --git a/cellprofiler_core/pipeline/io/_v6.py b/cellprofiler_core/pipeline/io/_v6.py index 70f0d2bd..8fc7775c 100644 --- a/cellprofiler_core/pipeline/io/_v6.py +++ b/cellprofiler_core/pipeline/io/_v6.py @@ -3,7 +3,7 @@ import os import re -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) import cellprofiler_core from cellprofiler_core.constants.pipeline import ( @@ -55,7 +55,7 @@ def load(pipeline, fd): pipeline_dict = json.load(fd) cp_version = int(re.sub(r"\.|rc\d", "", cellprofiler_core.__version__)) if cp_version != pipeline_dict['date_revision']: - logging.warning(f"Pipeline file is from a different version of CellProfiler. " + LOGGER.warning(f"Pipeline file is from a different version of CellProfiler. " f"Current:v{cp_version} File:v{pipeline_dict['date_revision']}." f" Will attempt to upgrade settings.") pipeline_modules = pipeline.modules(False) diff --git a/cellprofiler_core/preferences/__init__.py b/cellprofiler_core/preferences/__init__.py index e465807a..4bc12382 100644 --- a/cellprofiler_core/preferences/__init__.py +++ b/cellprofiler_core/preferences/__init__.py @@ -25,6 +25,9 @@ from ..constants.reader import ALL_READERS from ..utilities.image import image_resource + +LOGGER = logging.getLogger(__name__) + """get_absolute_path - mode = output. Assume "." is the default output dir""" ABSPATH_OUTPUT = "abspath_output" @@ -113,12 +116,12 @@ def get_config(): try: preferences_version_number = int(config_read(PREFERENCES_VERSION)) if preferences_version_number != PREFERENCES_VERSION_NUMBER: - logging.warning( + LOGGER.warning( "Preferences version mismatch: expected %d, at %d" % (PREFERENCES_VERSION_NUMBER, preferences_version_number) ) except: - logging.warning( + LOGGER.warning( "Preferences version was %s, not a number. Resetting to current version" % preferences_version_number ) @@ -302,7 +305,7 @@ def config_write_typed(key, value, key_type=None, flush=True): else: success = config.Write(key, value) if not success: - logging.error(f"Unable to write preference key {key}") + LOGGER.error(f"Unable to write preference key {key}") return if flush: config.Flush() @@ -319,7 +322,7 @@ def export_to_json(path): del config_dict[key] with open(path, mode='wt') as fd: json.dump(config_dict, fd) - logging.info(f"Wrote config to {path}") + LOGGER.info(f"Wrote config to {path}") def cell_profiler_root_directory(): @@ -842,10 +845,10 @@ def get_default_image_directory(): __default_image_directory = os.path.normcase(default_image_directory) return __default_image_directory except: - logging.error( + LOGGER.error( "Unknown failure when retrieving the default image directory", exc_info=True ) - logging.warning( + LOGGER.warning( "Warning: current path of %s is not a valid directory. Switching to home directory." % (default_image_directory.encode("ascii", "replace")) ) @@ -912,11 +915,11 @@ def get_default_output_directory(): __default_output_directory = os.path.normcase(default_output_directory) return __default_output_directory except: - logging.error( + LOGGER.error( "Unknown failure when retrieving the default output directory", exc_info=True, ) - logging.warning( + LOGGER.warning( "Warning: current path of %s is not a valid directory. Switching to home directory." % (default_output_directory.encode("ascii", "replace")) ) diff --git a/cellprofiler_core/readers/bioformats_reader.py b/cellprofiler_core/readers/bioformats_reader.py index 07c2f38c..1681c41b 100644 --- a/cellprofiler_core/readers/bioformats_reader.py +++ b/cellprofiler_core/readers/bioformats_reader.py @@ -1,21 +1,27 @@ import collections import itertools +import logging -import numpy +import numpy as np +import scyjava + +from ..utilities.image import is_file_url +from ..utilities.java import jimport from ..constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, MD_SIZE_Y, MD_SIZE_X, \ BIOFORMATS_IMAGE_EXTENSIONS from ..reader import Reader -from bioformats.formatreader import get_image_reader, release_image_reader, clear_image_reader_cache -from ..utilities.java import start_java +LOGGER = logging.getLogger(__name__) # bioformats returns 2 for these, imageio reader returns 3 SUPPORTED_EXTENSIONS = {'.tiff', '.tif', '.ome.tif', '.ome.tiff'} SEMI_SUPPORTED_EXTENSIONS = BIOFORMATS_IMAGE_EXTENSIONS -SUPPORTED_SCHEMES = {'file', 'http', 'https', 'ftp', 'ftps', 'omero', 's3'} +#TODO: disabled until CellProfiler/CellProfiler#4684 is resolved +# SUPPORTED_SCHEMES = {'file', 'http', 'https', 'ftp', 'ftps', 'omero', 's3'} +SUPPORTED_SCHEMES = {'file', 'http', 'https', 'ftp', 'ftps', 's3'} class BioformatsReader(Reader): """ @@ -31,26 +37,34 @@ class BioformatsReader(Reader): def __init__(self, image_file): self._reader = None - start_java() + self._is_file_open = False super().__init__(image_file) def get_reader(self): if self._reader is None: - - self._reader = get_image_reader(id(self), url=self.file.url) + ImageReader = jimport("loci.formats.ImageReader") + self._reader = ImageReader() + self._is_file_open = False + scyjava.when_jvm_stops(lambda: self._reader.close() if self._reader is not None else None) return self._reader + def _ensure_file_open(self): + if not self._is_file_open: + self.get_reader().setId(self.file.path) + self._is_file_open = True + def read(self, series=None, index=None, c=None, - z=None, - t=None, + z=0, + t=0, rescale=True, xywh=None, wants_max_intensity=False, channel_names=None, + XYWH=None, ): """Read a single plane from the image file. :param c: read from this channel. `None` = read color image if multichannel @@ -65,18 +79,140 @@ def read(self, :param wants_max_intensity: if `False`, only return the image; if `True`, return a tuple of image and max intensity :param channel_names: provide the channel names for the OME metadata + :param XYWH: a (x, y, w, h) tuple """ - reader = self.get_reader() - return reader.read( - c=c, - z=z, - t=t, - series=series, - index=index, - rescale=rescale, - wants_max_intensity=wants_max_intensity, - channel_names=channel_names, - ) + self._ensure_file_open() + + FormatTools = jimport("loci.formats.FormatTools") + ChannelSeparator = jimport("loci.formats.ChannelSeparator") + if series is not None: + self._reader.setSeries(series) + + if XYWH is not None: + assert isinstance(XYWH, tuple) and len(XYWH) == 4, "Invalid XYWH tuple" + openBytes_func = lambda x: self._reader.openBytes(x, XYWH[0], XYWH[1], XYWH[2], XYWH[3]) + width, height = XYWH[2], XYWH[3] + else: + openBytes_func = self._reader.openBytes + width, height = self._reader.getSizeX(), self._reader.getSizeY() + + # FIXME instead of np.frombuffer use scyjava.to_python, ideally that wraps memory + pixel_type = self._reader.getPixelType() + little_endian = self._reader.isLittleEndian() + if pixel_type == FormatTools.INT8: + dtype = np.int8 + scale = 255 + elif pixel_type == FormatTools.UINT8: + dtype = np.uint8 + scale = 255 + elif pixel_type == FormatTools.UINT16: + dtype = 'u2' + scale = 65535 + elif pixel_type == FormatTools.INT16: + dtype = 'i2' + scale = 65535 + elif pixel_type == FormatTools.UINT32: + dtype = 'u4' + scale = 2**32 + elif pixel_type == FormatTools.INT32: + dtype = 'i4' + scale = 2**32-1 + elif pixel_type == FormatTools.FLOAT: + dtype = 'f4' + scale = 1 + elif pixel_type == FormatTools.DOUBLE: + dtype = 'f8' + scale = 1 + max_sample_value = self._reader.getMetadataValue('MaxSampleValue') + if max_sample_value is not None: + try: + scale = scyjava.to_python(max_sample_value) + except: + LOGGER.warning("WARNING: failed to get MaxSampleValue for image. Intensities may be improperly scaled.") + if index is not None: + image = np.frombuffer(openBytes_func(index), dtype) + if len(image) / height / width in (3,4): + n_channels = int(len(image) / height / width) + if self._reader.isInterleaved(): + image.shape = (height, width, n_channels) + else: + image.shape = (n_channels, height, width) + image = image.transpose(1, 2, 0) + else: + image.shape = (height, width) + elif self._reader.isRGB() and self._reader.isInterleaved(): + index = self._reader.getIndex(z,0,t) + image = np.frombuffer(openBytes_func(index), dtype) + image.shape = (height, width, self._reader.getSizeC()) + if image.shape[2] > 3: + image = image[:, :, :3] + elif c is not None and self._reader.getRGBChannelCount() == 1: + index = self._reader.getIndex(z,c,t) + image = np.frombuffer(openBytes_func(index), dtype) + image.shape = (height, width) + elif self._reader.getRGBChannelCount() > 1: + n_planes = self._reader.getRGBChannelCount() + rdr = ChannelSeparator(self._reader) + planes = [ + np.frombuffer( + (rdr.openBytes(rdr.getIndex(z,i,t)) if XYWH is None else + rdr.openBytes(rdr.getIndex(z,i,t), XYWH[0], XYWH[1], XYWH[2], XYWH[3])), + dtype + ) for i in range(n_planes)] + + if len(planes) > 3: + planes = planes[:3] + elif len(planes) < 3: + # > 1 and < 3 means must be 2 + # see issue #775 + planes.append(np.zeros(planes[0].shape, planes[0].dtype)) + image = np.dstack(planes) + image.shape=(height, width, 3) + elif self._reader.getSizeC() > 1: + images = [ + np.frombuffer(openBytes_func(self._reader.getIndex(z,i,t)), dtype) + for i in range(self._reader.getSizeC())] + image = np.dstack(images) + image.shape = (height, width, self._reader.getSizeC()) + if not channel_names is None: + metadata = self._reader.getMetadataStore() + for i in range(self._reader.getSizeC()): + index = self._reader.getIndex(z, 0, t) + channel_name = metadata.getChannelName(index, i) + if channel_name is None: + channel_name = metadata.getChannelID(index, i) + channel_names.append(channel_name) + elif self._reader.isIndexed(): + # TODO can we use ChannelFiller, which automatically expands channels if they + # are "true" color and does nothing if they are "false" (e.g. applied color table)? + # + # The image data is indexes into a color lookup-table + # But sometimes the table is the identity table and just generates + # a monochrome RGB image + # + index = self._reader.getIndex(z,0,t) + image = np.frombuffer(openBytes_func(index),dtype) + lut = None + if pixel_type in (FormatTools.INT16, FormatTools.UINT16): + lut = self._reader.get16BitLookupTable() + else: + lut = self._reader.get8BitLookupTable() + if lut is not None: + lut = np.array([d for d in lut]).transpose() + image.shape = (height, width) + if (lut is not None) \ + and not np.all(lut == np.arange(lut.shape[0])[:, np.newaxis]): + image = lut[image, :] + else: + index = self._reader.getIndex(z,0,t) + image = np.frombuffer(openBytes_func(index),dtype) + image.shape = (height,width) + + if rescale: + image = image.astype(np.float32) / float(scale) + if wants_max_intensity: + return image, scale + return image def read_volume(self, series=None, @@ -91,8 +227,8 @@ def read_volume(self, # Whether a volume has planes stored in the z or t axis is often ambiguous. # If z-size > 1 we'll use z, else we'll use t. Otherwise user should choose # an axis to split in Images. - reader = self.get_reader() - bf_reader = reader.rdr + bf_reader = self.get_reader() + self._ensure_file_open() if series is None: series = 0 bf_reader.setSeries(series) @@ -111,7 +247,7 @@ def read_volume(self, t_range = [t] image_stack = [] for z_index, t_index in itertools.product(z_range, t_range): - data = reader.read( + data = self.read( series=series, c=c, z=z_index, @@ -122,7 +258,7 @@ def read_volume(self, channel_names=channel_names, ) image_stack.append(data) - return numpy.stack(image_stack) + return np.stack(image_stack) @classmethod def supports_format(cls, image_file, allow_open=False, volume=False): @@ -142,7 +278,6 @@ def supports_format(cls, image_file, allow_open=False, volume=False): The volume parameter specifies whether the reader will need to return a 3D array. .""" - url_lower = image_file.url.lower() if image_file.scheme not in SUPPORTED_SCHEMES: return -1 if image_file.scheme == 'omero': @@ -153,21 +288,12 @@ def supports_format(cls, image_file, allow_open=False, volume=False): if image_file.file_extension in SEMI_SUPPORTED_EXTENSIONS: return 3 return -1 - else: - try: - get_image_reader(id(image_file), url=image_file.url) - return 3 - except: - return -1 - - @classmethod - def clear_cached_readers(cls): - clear_image_reader_cache() def close(self): # If your reader opens a file, this needs to release any active lock, - release_image_reader(id(self)) - self._reader = None + if self._reader is not None and scyjava.jvm_started(): + self._reader.close() + self._reader = None def get_series_metadata(self): """Should return a dictionary with the following keys: @@ -181,7 +307,8 @@ def get_series_metadata(self): MD_SERIES_NAME - list of series names, one element per series. """ meta_dict = collections.defaultdict(list) - reader = self.get_reader().rdr + self._ensure_file_open() + reader = self.get_reader() series_count = reader.getSeriesCount() meta_dict[MD_SIZE_S] = series_count for i in range(series_count): diff --git a/cellprofiler_core/readers/ngff_reader.py b/cellprofiler_core/readers/ngff_reader.py index a04b0850..09d2580d 100644 --- a/cellprofiler_core/readers/ngff_reader.py +++ b/cellprofiler_core/readers/ngff_reader.py @@ -62,7 +62,7 @@ def get_reader(self): # Zarr has consolidated metadata. self._reader = zarr.convenience.open_consolidated(store, mode='r') else: - logging.warning(f"Image is on S3 but lacks consolidated metadata. " + LOGGER.warning(f"Image is on S3 but lacks consolidated metadata. " f"This may degrade reading performance. URL: {self.root}") self._reader = zarr.open(store, mode='r') elif not os.path.isdir(store.path): @@ -71,7 +71,7 @@ def get_reader(self): self._reader = zarr.open(store, mode='r') NGFFReader.ZARR_READER_CACHE[self.root] = self._reader, None if self.group: - logging.warning(f"Reader had a group? {self.group}") + LOGGER.warning(f"Reader had a group? {self.group}") return self._reader[self.group] return self._reader @@ -104,7 +104,7 @@ def read(self, return a tuple of image and max intensity :param channel_names: provide the channel names for the OME metadata """ - logging.debug(f"Reading {c=}, {z=}, {t=}, {series=}, {index=}, {xywh=}") + LOGGER.debug(f"Reading {c=}, {z=}, {t=}, {series=}, {index=}, {xywh=}") c2 = None if c is None else c + 1 z2 = None if z is None else z + 1 t2 = None if t is None else t + 1 diff --git a/cellprofiler_core/setting/_image_plane.py b/cellprofiler_core/setting/_image_plane.py index a28e0e02..251bc6ae 100644 --- a/cellprofiler_core/setting/_image_plane.py +++ b/cellprofiler_core/setting/_image_plane.py @@ -5,6 +5,8 @@ from ._validation_error import ValidationError +LOGGER = logging.getLogger(__name__) + class ImagePlane(Setting): """A setting that specifies an image plane @@ -55,7 +57,7 @@ def build(plane): if " " in url: # Spaces are not legal characters in URLs, nevertheless, I try # to accommodate - logging.warning( + LOGGER.warning( "URLs should not contain spaces. %s is the offending URL" % url ) url = url.replace(" ", "%20") diff --git a/cellprofiler_core/setting/range/_range.py b/cellprofiler_core/setting/range/_range.py index df558e48..e573252a 100644 --- a/cellprofiler_core/setting/range/_range.py +++ b/cellprofiler_core/setting/range/_range.py @@ -4,6 +4,8 @@ from .._validation_error import ValidationError +LOGGER = logging.getLogger(__name__) + class Range(Setting): """A setting representing a range between two values""" @@ -112,7 +114,7 @@ def set_value_text(self, value): self.__default_min = self.min self.__default_max = self.max except: - logging.debug("Illegal value in range setting: %s" % value) + LOGGER.debug("Illegal value in range setting: %s" % value) def test_valid(self, pipeline): values = self.value_text.split(",") diff --git a/cellprofiler_core/setting/text/number/_number.py b/cellprofiler_core/setting/text/number/_number.py index 50a68346..a93f728a 100644 --- a/cellprofiler_core/setting/text/number/_number.py +++ b/cellprofiler_core/setting/text/number/_number.py @@ -4,6 +4,8 @@ from ..._validation_error import ValidationError +LOGGER = logging.getLogger(__name__) + class Number(Text): """A setting that allows only numeric input """ @@ -55,7 +57,7 @@ def set_value_text(self, value_text): self.test_valid(None) self.__default = self.str_to_value(value_text) except: - logging.debug("Number set to illegal value: %s" % value_text) + LOGGER.debug("Number set to illegal value: %s" % value_text) def set_min_value(self, minval): """Programatically set the minimum value allowed""" diff --git a/cellprofiler_core/utilities/core/modules/__init__.py b/cellprofiler_core/utilities/core/modules/__init__.py index 40cfc6cf..13e7bec7 100644 --- a/cellprofiler_core/utilities/core/modules/__init__.py +++ b/cellprofiler_core/utilities/core/modules/__init__.py @@ -20,6 +20,8 @@ from ..plugins import load_plugins +LOGGER = logging.getLogger(__name__) + def check_module(module, name): if hasattr(module, "do_not_check"): return @@ -68,14 +70,14 @@ def add_module(mod, check_svn, class_name=None): cp_module = find_cpmodule(m) name = cp_module.module_name except Exception as e: - logging.warning("Could not load %s", mod, exc_info=True) + LOGGER.warning("Could not load %s", mod, exc_info=True) badmodules.append((mod, e)) return try: pymodules.append(m) if name in all_modules: - logging.warning( + LOGGER.warning( "Multiple definitions of module %s\n\told in %s\n\tnew in %s", name, sys.modules[all_modules[name].__module__].__file__, @@ -91,7 +93,7 @@ def add_module(mod, check_svn, class_name=None): if match is not None: svn_revisions[name] = match.groups()[0] except Exception as e: - logging.warning("Failed to load %s", name, exc_info=True) + LOGGER.warning("Failed to load %s", name, exc_info=True) badmodules.append((mod, e)) if name in all_modules: del all_modules[name] @@ -114,7 +116,7 @@ def add_module(mod, check_svn, class_name=None): add_module("cellprofiler.modules." + modname, True, class_name=classname) if len(badmodules) > 0: - logging.warning( + LOGGER.warning( "could not load these modules: %s", ",".join([x[0] for x in badmodules]) ) diff --git a/cellprofiler_core/utilities/hdf5_dict.py b/cellprofiler_core/utilities/hdf5_dict.py index 0e96eb40..5bb6db59 100644 --- a/cellprofiler_core/utilities/hdf5_dict.py +++ b/cellprofiler_core/utilities/hdf5_dict.py @@ -22,10 +22,12 @@ import cellprofiler_core.utilities.legacy -install_aliases() LOGGER = logging.getLogger(__name__) +install_aliases() + + version_number = 1 VERSION = "Version" @@ -183,7 +185,7 @@ def __init__( self.filename = hdf5_filename self.top_level_group_name = top_level_group_name LOGGER.debug( - "HDF5Dict.__init__(): %s, temporary=%s, copy=%s, mode=%s", + "%s, temporary=%s, copy=%s, mode=%s", self.filename, self.is_temporary, copy, @@ -363,7 +365,7 @@ def __init__( def __del__(self): LOGGER.debug( - "HDF5Dict.__del__(): %s, temporary=%s", self.filename, self.is_temporary + "%s, temporary=%s", self.filename, self.is_temporary ) self.close() diff --git a/cellprofiler_core/utilities/java.py b/cellprofiler_core/utilities/java.py index 01745c32..7d666398 100644 --- a/cellprofiler_core/utilities/java.py +++ b/cellprofiler_core/utilities/java.py @@ -4,80 +4,13 @@ import logging import os -import sys -import threading - -import javabridge -import prokaryote -from javabridge._javabridge import get_vm +import scyjava import cellprofiler_core.preferences -JAVA_STARTED = False - LOGGER = logging.getLogger(__name__) - -def get_jars(): - """ - Get the final list of JAR files passed to javabridge - """ - - class_path = [] - if "CLASSPATH" in os.environ: - class_path += os.environ["CLASSPATH"].split(os.pathsep) - LOGGER.debug( - "Adding Java class path from environment variable, " "CLASSPATH" "" - ) - LOGGER.debug(" CLASSPATH=" + os.environ["CLASSPATH"]) - - pathname = os.path.dirname(prokaryote.__file__) - - jar_files = [ - os.path.join(pathname, f) - for f in os.listdir(pathname) - if f.lower().endswith(".jar") - ] - - class_path += javabridge.JARS + jar_files - - if sys.platform.startswith("win") and not hasattr(sys, "frozen"): - # Have to find tools.jar - from javabridge.locate import find_jdk - - jdk_path = find_jdk() - if jdk_path is not None: - tools_jar = os.path.join(jdk_path, "lib", "tools.jar") - class_path.append(tools_jar) - else: - LOGGER.warning("Failed to find tools.jar") - return class_path - - -def find_logback_xml(): - """Find the location of the logback.xml file for Java logging config - - Paths to search are the current directory, the utilities directory - and ../../java/src/main/resources - """ - paths = [ - os.curdir, - os.path.dirname(__file__), - os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), - "java", - "src", - "main", - "resources", - ), - ] - for path in paths: - target = os.path.join(path, "logback.xml") - if os.path.isfile(target): - return target - - def start_java(): """Start CellProfiler's JVM via Javabridge @@ -88,28 +21,18 @@ def start_java(): cpprefs.get_awt_headless() - controls java.awt.headless to prevent awt from being invoked """ - thread_id = threading.get_ident() - if get_vm().is_active(): - javabridge.attach() + if scyjava.jvm_started(): return + + # Add Bio-Formats Java dependency. + scyjava.config.endpoints.append("ome:formats-gpl") + scyjava.config.endpoints.append("org.scijava:scijava-config") + LOGGER.info("Initializing Java Virtual Machine") args = [ "-Dloci.bioformats.loaded=true", - "-Djava.util.prefs.PreferencesFactory=" - + "org.cellprofiler.headlesspreferences.HeadlessPreferencesFactory", ] - logback_path = find_logback_xml() - - if logback_path is not None: - if sys.platform.startswith("win"): - logback_path = logback_path.replace("\\", "/") - if logback_path[1] == ":": - # \\localhost\x$ is same as x: - logback_path = "//localhost/" + logback_path[0] + "$" + logback_path[2:] - args.append("-Dlogback.configurationFile=%s" % logback_path) - - class_path = get_jars() awt_headless = cellprofiler_core.preferences.get_awt_headless() if awt_headless: LOGGER.debug("JVM will be started with AWT in headless mode") @@ -123,18 +46,19 @@ def start_java(): ) % os.environ["CP_JDWP_PORT"] ) - javabridge.start_vm(args=args, class_path=class_path) + scyjava.start_jvm(options=args) # # Enable Bio-Formats directory cacheing # - javabridge.attach() - c_location = javabridge.JClassWrapper("loci.common.Location") - c_location.cacheDirectoryListings(True) + Location = scyjava.jimport("loci.common.Location") + Location.cacheDirectoryListings(True) LOGGER.debug("Enabled Bio-formats directory cacheing") - global JAVA_STARTED - JAVA_STARTED = True def stop_java(): LOGGER.info("Shutting down Java Virtual Machine") - javabridge.kill_vm() + scyjava.shutdown_jvm() + +def jimport(package): + start_java() + return scyjava.jimport(package) diff --git a/cellprofiler_core/utilities/zmq/__init__.py b/cellprofiler_core/utilities/zmq/__init__.py index 85f2852c..173d0b91 100644 --- a/cellprofiler_core/utilities/zmq/__init__.py +++ b/cellprofiler_core/utilities/zmq/__init__.py @@ -18,6 +18,9 @@ Request, ) + +LOGGER = logging.getLogger(__name__) + NOTIFY_SOCKET_ADDR = "inproc://BoundaryNotifications" SD_KEY_DICT = "__keydict__" @@ -256,10 +259,10 @@ def lock_thread_fn(): request = msg[2] assert isinstance(request, LockStatusRequest) assert isinstance(boundary, Boundary) - logging.info("Received lock status request for %s" % request.uid) + LOGGER.info("Received lock status request for %s" % request.uid) reply = LockStatusReply(request.uid, request.uid in locked_uids) if reply.locked: - logging.info( + LOGGER.info( "Denied lock request for %s" % locked_uids[request.uid] ) boundary.enqueue_reply(request, reply) @@ -277,7 +280,7 @@ def lock_thread_fn(): except Exception as e: msg[3].put(e) __lock_thread = None - logging.info("Exiting the lock thread") + LOGGER.info("Exiting the lock thread") __lock_thread = threading.Thread(target=lock_thread_fn) __lock_thread.setName("FileLockThread") @@ -309,12 +312,12 @@ def lock_file(path, timeout=3): except OSError as e: if e.errno != errno.EEXIST: raise - logging.info("Lockfile for %s already exists - contacting owner" % path) + LOGGER.info("Lockfile for %s already exists - contacting owner" % path) with open(lock_path, "r") as f: remote_address = f.readline().strip() remote_uid = f.readline().strip() if len(remote_address) > 0 and len(remote_uid) > 0: - logging.info("Owner is %s" % remote_address) + LOGGER.info("Owner is %s" % remote_address) request_socket = the_boundary.zmq_context.socket(zmq.REQ) request_socket.setsockopt(zmq.LINGER, 0) assert isinstance(request_socket, zmq.Socket) @@ -333,7 +336,7 @@ def lock_file(path, timeout=3): lock_response = lock_request.recv(socket) if isinstance(lock_response, LockStatusReply): if lock_response.locked: - logging.info("%s is locked" % path) + LOGGER.info("%s is locked" % path) return False keep_polling = False # diff --git a/cellprofiler_core/utilities/zmq/_boundary.py b/cellprofiler_core/utilities/zmq/_boundary.py index dea4c0bc..c7d5ef9b 100644 --- a/cellprofiler_core/utilities/zmq/_boundary.py +++ b/cellprofiler_core/utilities/zmq/_boundary.py @@ -12,6 +12,8 @@ from ...constants.worker import NOTIFY_STOP, NOTIFY_RUN +LOGGER = logging.getLogger(__name__) + class Boundary: """This object serves as the interface between a ZMQ socket passing Requests and Replies, and a thread or threads serving those requests. @@ -89,7 +91,7 @@ def register_analysis(self, analysis_id, upward_queue): self.analysis_context = AnalysisContext( analysis_id, upward_queue, self.analysis_context_lock ) - logging.debug(f"Registered analysis as id {analysis_id}") + LOGGER.debug(f"Registered analysis as id {analysis_id}") def register_request_class(self, cls_request, upward_queue): """Register a queue to receive requests of the given class @@ -191,7 +193,7 @@ def spin(self): msg = selfnotify_socket.recv() # Let's watch out for stop signals anyway if msg == NOTIFY_STOP: - logging.warning("Captured a stop message over zmq") + LOGGER.warning("Captured a stop message over zmq") received_stop = True continue req = Communicable.recv(s, routed=True) @@ -203,7 +205,7 @@ def spin(self): q.put([self, self.NOTIFY_REQUEST, req]) break else: - logging.warning( + LOGGER.warning( "Received a request that wasn't an AnalysisRequest: %s" % str(type(req)) ) @@ -211,7 +213,7 @@ def spin(self): continue if s != request_socket: # Request is on the external socket - logging.warning("Received a request on the external socket") + LOGGER.warning("Received a request on the external socket") req.reply(BoundaryExited()) continue # @@ -251,13 +253,13 @@ def spin(self): request_socket.close() selfnotify_socket.close() - logging.debug("Shutting down the boundary thread") + LOGGER.debug("Shutting down the boundary thread") except: # # Pretty bad - a logic error or something extremely unexpected # We're close to hosed here, better die an ugly death. # - logging.critical("Unhandled exception in boundary thread.", + LOGGER.critical("Unhandled exception in boundary thread.", exc_info=True) import os os._exit(-1) diff --git a/cellprofiler_core/worker/__init__.py b/cellprofiler_core/worker/__init__.py index ed2688a1..3f706b88 100644 --- a/cellprofiler_core/worker/__init__.py +++ b/cellprofiler_core/worker/__init__.py @@ -34,6 +34,9 @@ from cellprofiler_core.preferences import set_always_continue, set_conserve_memory from cellprofiler_core.worker._worker import Worker + +LOGGER = logging.getLogger(__name__) + """Set the log level through the environment by specifying AW_LOG_LEVEL""" AW_LOG_LEVEL = "AW_LOG_LEVEL" @@ -80,6 +83,7 @@ def aw_parse_args(): parser.add_option( "--log-level", dest="log_level", + type="int", help="Logging level for logger: DEBUG, INFO, WARNING, ERROR", default=os.environ.get(AW_LOG_LEVEL, logging.INFO), ) @@ -122,7 +126,10 @@ def aw_parse_args(): logging.root.setLevel(options.log_level) if len(logging.root.handlers) == 0: - logging.root.addHandler(logging.StreamHandler()) + stream_handler = logging.StreamHandler() + fmt = logging.Formatter("%(process)d|%(levelno)s|%(name)s::%(funcName)s: %(message)s") + stream_handler.setFormatter(fmt) + logging.root.addHandler(stream_handler) if not options.work_server_address and options.work_server_address and \ options.analysis_id: @@ -144,7 +151,7 @@ def aw_parse_args(): if options.always_continue is not None: set_always_continue(options.always_continue, globally=False) else: - logging.warning("Plugins directory not set") + LOGGER.warning("Plugins directory not set") if __name__ == "__main__": @@ -249,14 +256,14 @@ def main(): ): worker_thread.join() the_zmq_context.destroy(linger=0) - logging.debug("Worker thread joined") + LOGGER.debug("Worker thread joined") # # Shutdown - need to handle some global cleanup here # try: stop_java() except: - logging.warning("Failed to stop the JVM", exc_info=True) + LOGGER.warning("Failed to stop the JVM", exc_info=True) def monitor_keepalive(context, keepalive_address): @@ -289,7 +296,7 @@ def monitor_keepalive(context, keepalive_address): event = keepalive_socket.poll(5000) if not event: missed += 1 - logging.warning(f"Worker failed to receive communication for" + LOGGER.warning(f"Worker failed to receive communication for" f" {5 * missed} seconds") else: missed = 0 @@ -301,12 +308,12 @@ def monitor_keepalive(context, keepalive_address): keepalive_socket.close() if missed >= 3: # Parent is dead, hard kill - logging.critical("Worker stopped receiving communication from " + LOGGER.critical("Worker stopped receiving communication from " "CellProfiler, shutting down now") else: # Stop signal captured, give some time to shut down gracefully. time.sleep(10) - logging.info("Cancelling worker") + LOGGER.info("Cancelling worker") # hard exit after 10 seconds unless app exits for m in all_measurements: @@ -314,7 +321,7 @@ def monitor_keepalive(context, keepalive_address): m.close() except: pass - logging.error("Worker failed to stop gracefully, forcing exit now") + LOGGER.error("Worker failed to stop gracefully, forcing exit now") os._exit(0) diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index 58de4404..c3da62f7 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -4,7 +4,6 @@ import time import traceback -import javabridge import zmq from ._pipeline_event_listener import PipelineEventListener @@ -25,7 +24,6 @@ from ..constants.worker import NOTIFY_STOP from ..constants.worker import all_measurements from ..measurement import Measurements -from ..utilities.java import JAVA_STARTED from ..utilities.measurement import load_measurements_from_buffer from ..pipeline import CancelledException from ..preferences import get_awt_headless @@ -34,13 +32,14 @@ from ..workspace import Workspace +LOGGER = logging.getLogger(__name__) + class Worker: """An analysis worker processing work at a given address """ def __init__(self, context, analysis_id, work_request_address, keepalive_address, with_stop_run_loop=True): - from bioformats.formatreader import set_omero_login_hook self.context = context self.work_request_address = work_request_address @@ -52,7 +51,9 @@ def __init__(self, context, analysis_id, work_request_address, keepalive_address self.preferences = None self.initial_measurements = None - set_omero_login_hook(self.omero_login_handler) + #TODO: disabled until CellProfiler/CellProfiler#4684 is resolved + # from ..bioformats.formatreader import set_omero_login_hook + # set_omero_login_hook(self.omero_login_handler) def __enter__(self): # pipeline listener object @@ -94,15 +95,12 @@ def __exit__(self, type, value, tb): self.worker.exit_thread() def enter_thread(self): - if not get_awt_headless(): - javabridge.activate_awt() + LOGGER.debug("Entering worker thread") def exit_thread(self): from cellprofiler_core.constants.reader import ALL_READERS for reader in ALL_READERS.values(): reader.clear_cached_readers() - if JAVA_STARTED: - javabridge.detach() def run(self): from cellprofiler_core.pipeline.event import CancelledException @@ -110,7 +108,7 @@ def run(self): with self.AnalysisWorkerThreadObject(self): while not self.cancelled: try: - logging.debug("Requesting a job") + LOGGER.debug("Requesting a job") # fetch a job the_request = Work(self.current_analysis_id) job = self.send(the_request) @@ -134,26 +132,26 @@ def do_job(self, job): try: send_dictionary = job.wants_dictionary - logging.info("Starting job") + LOGGER.info("Starting job") # Fetch the pipeline and preferences for this analysis if we don't have it current_pipeline = self.pipeline current_preferences = self.preferences if not current_pipeline: - logging.debug("Fetching pipeline and preferences") + LOGGER.debug("Fetching pipeline and preferences") rep = self.send(PipelinePreferences(self.current_analysis_id)) - logging.debug("Received pipeline and preferences response") + LOGGER.debug("Received pipeline and preferences response") preferences_dict = rep.preferences # update preferences to match remote values set_preferences_from_dict(preferences_dict) - logging.debug("Loading pipeline") + LOGGER.debug("Loading pipeline") current_pipeline = cpp.Pipeline() pipeline_chunks = rep.pipeline_blob.tolist() pipeline_io = io.StringIO("".join(pipeline_chunks)) current_pipeline.loadtxt(pipeline_io, raise_on_error=True) - logging.debug("Pipeline loaded") + LOGGER.debug("Pipeline loaded") current_pipeline.add_listener(self.pipeline_listener.handle_event) current_preferences = rep.preferences self.pipeline = current_pipeline @@ -165,16 +163,16 @@ def do_job(self, job): # Reset the listener's state self.pipeline_listener.reset() - logging.debug("Getting initial measurements") + LOGGER.debug("Getting initial measurements") # Fetch the path to the intial measurements if needed. if self.initial_measurements is None: - logging.debug("Sending initial measurements request") + LOGGER.debug("Sending initial measurements request") rep = self.send(InitialMeasurements(self.current_analysis_id)) - logging.debug("Got initial measurements") + LOGGER.debug("Got initial measurements") self.initial_measurements = load_measurements_from_buffer(rep.buf) else: - logging.debug("Has initial measurements") + LOGGER.debug("Has initial measurements") # Make a copy of the measurements for writing during this job current_measurements = Measurements(copy=self.initial_measurements) all_measurements.add(current_measurements) @@ -183,7 +181,7 @@ def do_job(self, job): successful_image_set_numbers = [] image_set_numbers = job.image_set_numbers worker_runs_post_group = job.worker_runs_post_group - logging.info("Doing job: " + ",".join(map(str, image_set_numbers))) + LOGGER.info("Doing job: " + ",".join(map(str, image_set_numbers))) self.pipeline_listener.image_set_number = image_set_numbers[0] @@ -257,11 +255,11 @@ def do_job(self, job): ) rep = self.send(req) except CancelledException: - logging.info("Aborting job after cancellation") + LOGGER.info("Aborting job after cancellation") abort = True except Exception as e: try: - logging.error("Error in pipeline", exc_info=True) + LOGGER.error("Error in pipeline", exc_info=True) if ( self.handle_exception(image_set_number=image_set_number) == ED_STOP @@ -269,7 +267,7 @@ def do_job(self, job): abort = True break except: - logging.error( + LOGGER.error( "Error in handling of pipeline exception", exc_info=True ) # this is bad. We can't handle nested exceptions @@ -308,7 +306,7 @@ def do_job(self, job): raise except Exception: - logging.error("Error in worker", exc_info=True) + LOGGER.error("Error in worker", exc_info=True) if self.handle_exception() == ED_STOP: raise CancelledException("Cancelling after user-requested stop") finally: @@ -359,13 +357,14 @@ def post_group_display_handler(self, module, display_data, image_set_number): ) rep = self.send(req) - def omero_login_handler(self): - """Handle requests for an Omero login""" - from bioformats.formatreader import use_omero_credentials + #TODO: disabled until CellProfiler/CellProfiler#4684 is resolved + # def omero_login_handler(self): + # """Handle requests for an Omero login""" + # from cellprofiler_core.bioformats.formatreader import use_omero_credentials - req = OmeroLogin(self.current_analysis_id) - rep = self.send(req) - use_omero_credentials(rep.credentials) + # req = OmeroLogin(self.current_analysis_id) + # rep = self.send(req) + # use_omero_credentials(rep.credentials) def send(self, req, work_socket=None): """Send a request and receive a reply @@ -394,14 +393,14 @@ def send(self, req, work_socket=None): elif socket == self.keepalive_socket: notify_msg = self.keepalive_socket.recv() if notify_msg == NOTIFY_STOP: - logging.debug("Worker received cancel notification") + LOGGER.debug("Worker received cancel notification") self.cancelled = True self.raise_cancel( "Received stop notification while waiting for " "response from %s" % str(req) ) else: - logging.error("Unexpected message on keepalive: " + notify_msg.decode()) + LOGGER.error("Unexpected message on keepalive: " + notify_msg.decode()) elif socket == work_socket: response = req.recv(work_socket) if isinstance(response, (UpstreamExit, ServerExited)): @@ -422,7 +421,7 @@ def raise_cancel(self, msg="Cancelling analysis"): """ from cellprofiler_core.pipeline.event import CancelledException - logging.debug(msg) + LOGGER.debug(msg) self.cancelled = True if self.initial_measurements is not None: self.initial_measurements.close() diff --git a/cellprofiler_core/workspace/_workspace.py b/cellprofiler_core/workspace/_workspace.py index 116de774..2a96ff6c 100644 --- a/cellprofiler_core/workspace/_workspace.py +++ b/cellprofiler_core/workspace/_workspace.py @@ -14,6 +14,8 @@ from ..utilities.hdf5_dict import HDF5FileList +LOGGER = logging.getLogger(__name__) + class Workspace: """The workspace contains the processing information and state for a pipeline run on an image set @@ -536,7 +538,7 @@ def refresh_image_set(self, force=False): result = self.pipeline.prepare_run(self, stop_module) return result except: - logging.error("Failed during prepare_run", exc_info=1) + LOGGER.error("Failed during prepare_run", exc_info=1) return False finally: if no_image_set_list: @@ -561,7 +563,7 @@ def notify(self, event): try: callback(event) except: - logging.error("Notification callback threw an exception", exc_info=1) + LOGGER.error("Notification callback threw an exception", exc_info=1) def __on_file_list_changed(self): self.notify(self.WorkspaceFileListNotification(self)) diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..420e5e99 --- /dev/null +++ b/environment.yml @@ -0,0 +1,18 @@ +# run: conda env create +name: cpcore-dev +channels: + - conda-forge +dependencies: + - boto3 + - docutils + - future + - h5py + - matplotlib + - numpy + - psutil + - pyzmq + - scikit-image + - scipy + - scyjava + - pip: + - centrosome diff --git a/setup.py b/setup.py index 2110e179..40e306c6 100644 --- a/setup.py +++ b/setup.py @@ -24,18 +24,17 @@ "boto3>=1.12.28", "centrosome==1.2.1", "docutils==0.15.2", + "future>=0.18.2", "fsspec>=2021.11.0", "h5py~=3.6.0", "lxml>=4.6.4", "matplotlib>=3.1.3", "numpy>=1.18.2", - "prokaryote==2.4.4", "psutil>=5.7.0", - "python-bioformats==4.0.6", - "python-javabridge==4.0.3", "pyzmq~=22.3", "scikit-image>=0.16.2", "scipy>=1.4.1", + "scyjava>=1.6.0", "zarr>=2.10", "google-cloud-storage>=2.6.0", ], diff --git a/tests/analysis/test_analysis.py b/tests/analysis/test_analysis.py index 551b37ca..91ce1092 100644 --- a/tests/analysis/test_analysis.py +++ b/tests/analysis/test_analysis.py @@ -3,31 +3,25 @@ import logging import queue - import pytest -from importlib.util import find_spec - -import cellprofiler_core.constants.measurement -import cellprofiler_core.utilities.measurement -from cellprofiler_core.analysis._analysis import Analysis -from cellprofiler_core.analysis._runner import Runner -from cellprofiler_core.analysis.reply import ImageSetSuccess - -logger = logging.getLogger(__name__) -# logger.addHandler(logging.StreamHandler()) -logger.setLevel(logging.DEBUG) -import six.moves import inspect import numpy import os -import six.moves.queue import tempfile import threading import traceback import unittest import uuid import zmq +import six.moves +import six.moves.queue +from importlib.util import find_spec +import cellprofiler_core.constants.measurement +import cellprofiler_core.utilities.measurement +from cellprofiler_core.analysis._analysis import Analysis +from cellprofiler_core.analysis._runner import Runner +from cellprofiler_core.analysis.reply import ImageSetSuccess import cellprofiler_core.analysis import cellprofiler_core.analysis.request as anarequest import cellprofiler_core.analysis.reply as anareply @@ -39,6 +33,9 @@ import cellprofiler_core.utilities.zmq import cellprofiler_core.utilities.zmq.communicable.reply.upstream_exit + +LOGGER = logging.getLogger(__name__) + IMAGE_NAME = "imagename" OBJECTS_NAME = "objectsname" IMAGE_FEATURE = "imagefeature" @@ -84,7 +81,7 @@ def __exit__(self, type, value, traceback): self.join() def run(self): - logger.info("Client thread starting") + LOGGER.info("Client thread starting") try: self.work_socket = self.zmq_context.socket(zmq.REQ) self.recv_notify_socket = self.zmq_context.socket(zmq.SUB) @@ -116,17 +113,17 @@ def run(self): traceback.print_exc() self.response_queue.put((e, None)) except: - logger.warning("Client thread caught exception", exc_info=True) + LOGGER.warning("Client thread caught exception", exc_info=True) self.start_signal.release() finally: - logger.debug("Client thread exiting") + LOGGER.debug("Client thread exiting") def stop(self): self.keep_going = False self.notify_socket.send(b"Stop") def send(self, req): - logger.debug(" Enqueueing send of %s" % str(type(req))) + LOGGER.debug(" Enqueueing send of %s" % str(type(req))) self.queue.put((self.do_send, req)) self.notify_socket.send(b"Send") return self.recv @@ -143,7 +140,7 @@ def request_work(self): ) def do_send(self, req): - logger.info(" Sending %s" % str(type(req))) + LOGGER.info(" Sending %s" % str(type(req))) cellprofiler_core.utilities.zmq.communicable.Communicable.send( req, self.work_socket ) @@ -156,7 +153,7 @@ def do_send(self, req): if not self.keep_going: raise Exception("Cancelled") if socks.get(self.work_socket, None) == zmq.POLLIN: - logger.debug(" Received response for %s" % str(type(req))) + LOGGER.debug(" Received response for %s" % str(type(req))) return cellprofiler_core.utilities.zmq.communicable.Communicable.recv( self.work_socket ) @@ -164,13 +161,13 @@ def do_send(self, req): self.poller.unregister(self.work_socket) def recv(self): - logger.debug(" Waiting for client thread") + LOGGER.debug(" Waiting for client thread") exception, result = self.response_queue.get() if exception is not None: - logger.debug(" Client thread communicated exception") + LOGGER.debug(" Client thread communicated exception") raise exception else: - logger.debug(" Client thread communicated result") + LOGGER.debug(" Client thread communicated result") return result def listen_for_heartbeat(self, address): @@ -194,7 +191,7 @@ def do_listen_for_heartbeat(self, address): beat = self.keepalive_socket.recv() return beat except: - logger.info("Failed to listen") + LOGGER.info("Failed to listen") finally: self.poller.unregister(self.keepalive_socket) self.keepalive_socket.close() @@ -322,7 +319,7 @@ def check_display_post_run_requests(self, pipeline): self.assertEqual(result.module_num, module.module_num) def test_01_01_start_and_stop(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.make_pipeline_and_measurements_and_start() @@ -338,7 +335,7 @@ def test_01_01_start_and_stop(self): self.assertIsInstance( analysis_finished.measurements, cellprofiler_core.measurement.Measurements ) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -346,7 +343,7 @@ def test_02_01_heartbeat(self): """The heartbeat should send b'KeepAlive' each second. When the analysis is cancelled or finishes we should see b'Stop' as the final transmission on the socket""" - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -379,7 +376,7 @@ def collect_messages(container): assert not spy_thread.is_alive() assert len(messages) assert messages[-1] == NOTIFY_STOP - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -392,12 +389,12 @@ def test_03_01_get_work(self): self.assertSequenceEqual(response.image_set_numbers, (1,)) self.assertFalse(response.worker_runs_post_group) self.assertTrue(response.wants_dictionary) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_03_02_get_work_twice(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -408,12 +405,12 @@ def test_03_02_get_work_twice(self): self.assertIsInstance(response, anareply.Work) response = worker.send(anarequest.Work(worker.analysis_id))() self.assertIsInstance(response, anareply.NoWork) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_03_03_cancel_before_work(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -431,13 +428,13 @@ def test_03_03_cancel_before_work(self): # response, # cellprofiler_core.utilities.zmq.communicable.reply.upstream_exit.BoundaryExited, # ) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) # FIXME: wxPython 4 PR # def test_04_01_pipeline_preferences(self): - # logger.debug("Entering %s" % inspect.getframeinfo(inspect.currentframe()).function) + # LOGGER.debug("Entering %s" % inspect.getframeinfo(inspect.currentframe()).function) # pipeline, m = self.make_pipeline_and_measurements_and_start() # cellprofiler_core.preferences.set_headless() # title_font_name = "Rosewood Std Regular" @@ -477,10 +474,10 @@ def test_03_03_cancel_before_work(self): # self.assertEqual(preferences[cellprofiler_core.preferences.DEFAULT_OUTPUT_DIRECTORY], # cellprofiler_core.preferences.get_default_output_directory()) # - # logger.debug("Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function) + # LOGGER.debug("Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function) def test_04_02_initial_measurements_request(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -522,12 +519,12 @@ def test_04_02_initial_measurements_request(self): numpy.testing.assert_almost_equal(sv, cv) finally: client_measurements.close() - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_03_interaction(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -543,12 +540,12 @@ def test_04_03_interaction(self): reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Interaction) self.assertEqual(reply.hello, "world") - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_04_01_display(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -567,12 +564,12 @@ def test_04_04_01_display(self): reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Ack) self.assertEqual(reply.message, "Gimme Pony") - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_04_02_display_post_group(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -592,12 +589,12 @@ def test_04_04_02_display_post_group(self): reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Ack) self.assertEqual(reply.message, "Gimme Pony") - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_04_05_exception(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start() @@ -647,7 +644,7 @@ def test_04_05_exception(self): request.reply(anareply.Ack()) reply = fn_interaction_reply() self.assertIsInstance(reply, anareply.Ack) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -661,7 +658,7 @@ def test_05_01_imageset_with_dictionary(self): # request.Work (with spin until WorkReply received) # request.SharedDictionary # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start(nimage_sets=2) @@ -690,12 +687,12 @@ def test_05_01_imageset_with_dictionary(self): self.assertCountEqual(list(ed.keys()), list(d.keys())) for k in list(ed.keys()): numpy.testing.assert_almost_equal(ed[k], d[k]) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) def test_05_02_groups(self): - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) pipeline, m = self.make_pipeline_and_measurements_and_start( @@ -716,7 +713,7 @@ def test_05_02_groups(self): self.assertSequenceEqual(response.image_set_numbers, [3, 4]) self.assertTrue(response.worker_runs_post_group) self.assertFalse(response.wants_dictionary) - logger.debug( + LOGGER.debug( "Exiting %s" % inspect.getframeinfo(inspect.currentframe()).function ) @@ -725,7 +722,7 @@ def test_06_01_single_imageset(self): # Test a full cycle of analysis with an image set list # with a single image set # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -801,7 +798,7 @@ def test_06_01_single_imageset(self): def test_06_02_test_three_imagesets(self): # Test an analysis of three imagesets # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -907,7 +904,7 @@ def test_06_02_test_three_imagesets(self): def test_06_03_test_grouped_imagesets(self): # Test an analysis of four imagesets in two groups # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -998,7 +995,7 @@ def test_06_03_test_grouped_imagesets(self): def test_06_04_test_restart(self): # Test a restart of an analysis # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -1121,7 +1118,7 @@ def test_06_05_test_grouped_restart(self): # Test an analysis of four imagesets in two groups with all but one # complete. # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -1202,7 +1199,7 @@ def test_06_06_relationships(self): # # Test a transfer of the relationships table. # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True @@ -1316,7 +1313,7 @@ def test_06_07_worker_cancel(self): # # Test worker sending AnalysisCancelRequest # - logger.debug( + LOGGER.debug( "Entering %s" % inspect.getframeinfo(inspect.currentframe()).function ) self.wants_analysis_finished = True diff --git a/tests/conftest.py b/tests/conftest.py index 6525c127..e4efa53a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,3 @@ -import bioformats.formatreader - import cellprofiler_core.preferences import cellprofiler_core.utilities.java @@ -9,8 +7,6 @@ def pytest_sessionstart(session): def pytest_sessionfinish(session, exitstatus): - bioformats.formatreader.clear_image_reader_cache() - cellprofiler_core.utilities.java.stop_java() return exitstatus diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py index 3fda2a42..d7a0ca11 100644 --- a/tests/modules/__init__.py +++ b/tests/modules/__init__.py @@ -1,18 +1,16 @@ import os import tempfile - -import cellprofiler_core.utilities.legacy - -from bioformats import PT_UINT16 -from bioformats.formatwriter import write_image import logging import functools import hashlib - -logger = logging.getLogger(__name__) import numpy import unittest from urllib.request import URLopener +import skimage.io + +import cellprofiler_core.utilities.legacy + +LOGGER = logging.getLogger(__name__) __temp_example_images_folder = None __temp_test_images_folder = None @@ -38,7 +36,7 @@ def example_images_directory(): return path if __temp_example_images_folder is None: __temp_example_images_folder = tempfile.mkdtemp(prefix="cp_exampleimages") - logger.warning( + LOGGER.warning( "Creating temporary folder %s for example images" % __temp_example_images_folder ) @@ -60,7 +58,7 @@ def testimages_directory(): return path if __temp_test_images_folder is None: __temp_test_images_folder = tempfile.mkdtemp(prefix="cp_testimages") - logger.warning( + LOGGER.warning( "Creating temporary folder %s for test images" % __temp_test_images_folder ) return __temp_test_images_folder @@ -125,7 +123,11 @@ def make_12_bit_image(folder, filename, shape): if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) - write_image(path, img, PT_UINT16) + if len(shape) > 2: + skimage.io.imsave(path, numpy.transpose(img, (2,0,1)), imagej=True) + else: + skimage.io.imsave(path, img) + # # Now go through the file and find the TIF bits per sample IFD (#258) and # change it from 16 to 12. diff --git a/tests/modules/test_loaddata.py b/tests/modules/test_loaddata.py index 49fd7b5f..24f03129 100644 --- a/tests/modules/test_loaddata.py +++ b/tests/modules/test_loaddata.py @@ -4,10 +4,9 @@ import re import tempfile -import bioformats -import bioformats.formatreader import numpy import pytest +import skimage.io from cellprofiler_core.constants.measurement import COLTYPE_FLOAT from cellprofiler_core.constants.measurement import COLTYPE_INTEGER @@ -712,8 +711,6 @@ def test_scaling(): """Test loading an image scaled and unscaled""" folder = "loaddata" file_name = "1-162hrh2ax2.tif" - if not cellprofiler_core.utilities.java.JAVA_STARTED: - start_java() path = tests.modules.make_12_bit_image(folder, file_name, (22, 18)) csv_text = ( "Image_PathName_MyFile,Image_FileName_MyFile\n" "%s,%s\n" % os.path.split(path) @@ -749,8 +746,7 @@ def test_load_objects(): r.seed(1101) labels = r.randint(0, 10, size=(30, 20)).astype(numpy.uint8) handle, name = tempfile.mkstemp(".png") - start_java() - bioformats.write_image(name, labels, bioformats.PT_UINT8) + skimage.io.imsave(name, labels) os.close(handle) png_path, png_file = os.path.split(name) sbs_dir = os.path.join(get_data_directory(), "ExampleSBSImages") @@ -798,7 +794,6 @@ def test_load_objects(): value = measurements.get_current_measurement(OBJECTS_NAME, feature) assert len(value) == 9 finally: - bioformats.formatreader.clear_image_reader_cache() os.remove(name) os.remove(csv_name) diff --git a/tests/workspace/test_workspace.py b/tests/workspace/test_workspace.py index e9f12b36..7b050ce3 100644 --- a/tests/workspace/test_workspace.py +++ b/tests/workspace/test_workspace.py @@ -10,8 +10,8 @@ import cellprofiler_core.workspace from cellprofiler_core.utilities.hdf5_dict import FILE_LIST_GROUP, TOP_LEVEL_GROUP_NAME -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class TestWorkspace: def setup_method(self): @@ -22,7 +22,7 @@ def teardown_method(self): try: os.remove(path) except OSError: - logger.warning("Failed to close file %s" % path, exc_info=1) + LOGGER.warning("Failed to close file %s" % path, exc_info=1) def make_workspace_file(self): """Make a very basic workspace file"""