From 5e9638b0d7c54d095432d57270086c1f4a4a9dd3 Mon Sep 17 00:00:00 2001 From: mattloose Date: Wed, 3 Apr 2024 08:26:25 +0100 Subject: [PATCH 01/16] Introducing a dorado specific plugin which handles the change from ont_pyguppy_client_lib to ont_pybasecaller_client_lib --- src/readfish/_config.py | 1 + src/readfish/plugins/dorado.py | 274 +++++++++++++++++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 src/readfish/plugins/dorado.py diff --git a/src/readfish/_config.py b/src/readfish/_config.py index 343eaf56..227daeec 100644 --- a/src/readfish/_config.py +++ b/src/readfish/_config.py @@ -171,6 +171,7 @@ def load_module(self, override=False): """ builtins = { "guppy": "guppy", + "dorado": "dorado", "mappy": "mappy", "mappy_rs": "mappy_rs", "no_op": "_no_op", diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py new file mode 100644 index 00000000..f9842de5 --- /dev/null +++ b/src/readfish/plugins/dorado.py @@ -0,0 +1,274 @@ +"""Dorado plugin module + +Extension of pyBaseCaller that maintains a connection to the basecaller +""" +from __future__ import annotations +import logging +import os +import time +from collections import namedtuple +from pathlib import Path +from typing import Iterable, TYPE_CHECKING + +import numpy as np +import numpy.typing as npt +from minknow_api.protocol_pb2 import ProtocolRunInfo +from pybasecall_client_lib.helper_functions import package_read +from pybasecall_client_lib.pyclient import PyBasecallClient + +from readfish._loggers import setup_logger +from readfish.plugins.abc import CallerABC +from readfish.plugins.utils import Result +from readfish._utils import nice_join + + +if TYPE_CHECKING: + import minknow_api + +__all__ = ["Caller"] + +logger = logging.getLogger("RU_basecaller") +CALIBRATION = namedtuple("calibration", "scaling offset") + + +class DefaultDAQValues: + """Provides default calibration values + + Mimics the read_until_api calibration dict value from + https://github.com/nanoporetech/read_until_api/blob/2319bbe/read_until/base.py#L34 + all keys return scaling=1.0 and offset=0.0 + """ + + calibration = CALIBRATION(1.0, 0.0) + + def __getitem__(self, _): + return self.calibration + + +_DefaultDAQValues = DefaultDAQValues() + + +class Caller(CallerABC): + def __init__( + self, run_information: ProtocolRunInfo = None, debug_log=None, **kwargs + ): + self.logger = setup_logger("readfish_dorado_logger", log_file=debug_log) + self.supported_barcode_kits = None + self.supported_basecall_models = None + self.run_information = run_information + + # Set our own priority + self.dorado_params = kwargs + self.dorado_params["priority"] = PyBasecallClient.high_priority + # Set our own client name to appear in the dorado server logs + self.dorado_params["client_name"] = "Readfish_connection" + self.validate() + self.caller = PyBasecallClient(**self.dorado_params) + self.caller.connect() + + + def validate(self) -> None: + """Validate the parameters passed to Dorado to ensure they will initialise py Basecall Client correctly + + Currently checks: + 1. That the socket file exists + 2. That the Socket file has the correct permissions + 3. That the version of py basecall client lib installed matches the system version + + :return: None, if the parameters pass all the checks + + """ + for key in ("address", "config"): + if key not in self.dorado_params: + raise KeyError( + f"Required `caller_settings.dorado` {key} was not found in provided TOML. Please add." + ) + if self.dorado_params["address"].startswith("ipc://"): + # User is attempting to connect to an IPC socket + socket_path = Path(self.dorado_params["address"][6:]) + if not socket_path.exists(): + raise FileNotFoundError( + f"The provided dorado base-caller socket address doesn't appear to exist. Please check your dorado Settings. {self.dorado_params['address']}" + ) + + # check user permissions: + if not os.access(socket_path, os.R_OK): + raise RuntimeError( + f"The user account running readfish doesn't appear to have permissions to read the dorado base-caller socket. Please check permissions on {self.dorado_params['address']}. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information." + ) + if not os.access(socket_path, os.W_OK): + raise RuntimeError( + f"The user account running readfish doesn't appear to have permissions to write to the dorado base-caller socket. Please check permissions on {self.dorado_params['address']}. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information." + ) + # If we are connected to a live run, test if the base-caller model is acceptable. + # Connected to a live run via the minknow_api - get supported basecall and barcoding kits from the run info. + # Check them against provided values + if self.run_information is not None: + tags = self.run_information.meta_info.tags + + self.supported_basecall_models = tags[ + "available basecall models" + ].array_value + # Make a CSV str a list of strings, removing quotes and square brackets + if self.supported_basecall_models and isinstance( + self.supported_basecall_models, str + ): + self.supported_basecall_models = ( + tags["available basecall models"] + .array_value[1:-1] + .replace('"', "") + .split(",") + ) + # Faff on with sorting out available barcoding kits + # See https://github.com/nanoporetech/minknow_api/blob/829dbe8ac8e49efdf268d385b50440c52473188b/python/minknow_api/tools/protocols.py#L97C1-L97C7 + self.supported_barcode_kits = tags["barcoding kits"].array_value + # workaround for the set of barcoding kits being returned as a string rather + # that array of strings + if self.supported_barcode_kits and isinstance( + self.supported_barcode_kits, str + ): + self.supported_barcode_kits = ( + tags["barcoding kits"].array_value[1:-1].replace('"', "").split(",") + ) + + if tags["barcoding"].bool_value: + self.supported_barcode_kits.append(tags["kit"].string_value) + # If we are connected to a live run, and have suitable base calling models check the base-caller model is suitable for the flowcell and kit + if ( + self.supported_basecall_models + and f"{self.dorado_params['config'].replace('.cfg', '')}.cfg" + not in self.supported_basecall_models + ): + raise RuntimeError( + """The {} base-calling config listed in the readfish config TOML is not suitable for this flowcell and kit combination. + Please check the dorado value in the caller_settings.dorado section of your TOML file. + The following models are are given by ONT as suitable for this flow cell/kit combo:\n\t{}""".format( + self.dorado_params["config"], + nice_join( + self.supported_basecall_models, + sep="\n\t", + conjunction="and", + ), + ) + ) + + # If we are barcoding and have connected to a live run - try checking the listed barcode kit works with the flowcell and kit + if ( + barcoding_kits := self.dorado_params.get("barcode_kits", None) + ) is not None: + barcoding_kits = barcoding_kits.split() + if barcoding_kits and not set(barcoding_kits).issubset( + self.supported_barcode_kits + ): + raise RuntimeError( + "Barcoding kits specified in TOML {} not amongst those supported by the selected kit and protocol.\nSupported kits are:\n\t{}".format( + nice_join(barcoding_kits, conjunction="and"), + nice_join( + self.supported_barcode_kits, sep="\n\t", conjunction="and" + ), + ), + ) + return None + + def disconnect(self) -> None: + """Call the disconnect method on the PyBasecallClient""" + return self.caller.disconnect() + + def basecall( + self, + reads: Iterable[tuple[int, minknow_api.data_pb2.GetLiveReadsResponse.ReadData]], + signal_dtype: npt.DTypeLike, + daq_values: dict[int, namedtuple] = None, + ): + """Basecall live data from minknow RPC + + :param reads: List or generator of tuples containing (channel, MinKNOW.rpc.Read) + :param signal_dtype: Numpy dtype of the raw data + :param daq_values: Dictionary mapping channel to offset and scaling values. + If not provided default values of 1.0 and 0.0 are used. + :yield: + :rtype: readfish.plugins.utils.Result + """ + # FIXME: Occasionally dorado can report a read as not sent when it is + # successfully sent. Therefore we capture not sent reads + cache, skipped = {}, {} + reads_received, reads_sent = 0, 0 + daq_values = _DefaultDAQValues if daq_values is None else daq_values + + for channel, read in reads: + # Attach the "RF-" prefix + read_id = f"RF-{read.id}" + t0 = time.time() + cache[read_id] = (channel, read.number, t0) + + success = self.caller.pass_read( + package_read( + read_id=read_id, + raw_data=np.frombuffer(read.raw_data, signal_dtype), + daq_offset=daq_values[channel].offset, + daq_scaling=daq_values[channel].scaling, + # ToDo: Sample rate needs to be derived from the MinKNOW API + sampling_rate=float(5000), + start_time=int(read.start_sample), + ) + ) + if not success: + logging.warning(f"Could not send read {read_id!r} to Dorado") + # FIXME: This is resolved in later versions of dorado. + skipped[read_id] = cache.pop(read_id) + continue + else: + reads_sent += 1 + + sleep_time = self.caller.throttle - t0 + if sleep_time > 0: + time.sleep(sleep_time) + + while reads_received < reads_sent: + results = self.caller.get_completed_reads() + # TODO: incorporate time_received into logging? + # time_received = time.time() + + if not results: + time.sleep(self.caller.throttle) + continue + + for res_batch in results: + for res in res_batch: + read_id = res["metadata"]["read_id"] + try: + channel, read_number, time_sent = cache.pop(read_id) + except KeyError: + # FIXME: This is resolved in later versions of dorado. + channel, read_number, time_sent = skipped.pop(read_id) + reads_sent += 1 + res["metadata"]["read_id"] = read_id[3:] + self.logger.debug( + "@%s ch=%s\n%s\n+\n%s", + res["metadata"]["read_id"], + channel, + res["datasets"]["sequence"], + res["datasets"]["qstring"], + ) + barcode = res["metadata"].get("barcode_arrangement", None) + # TODO: Add Filter here + yield Result( + channel=channel, + read_number=read_number, + read_id=res["metadata"]["read_id"], + seq=res["datasets"]["sequence"], + barcode=barcode if barcode else None, + basecall_data=res, + ) + reads_received += 1 + + def describe(self) -> str: + """ + Describe the Dorado Caller + + :return: Description of parameters passed to this Dorado Caller plugin + """ + description = ["Utilising the Dorado base-caller plugin:"] + for param in self.dorado_params.keys(): + description.append(f"\t- {param}: {self.dorado_params[param]}") + return "\n".join(description) From 0ad365adcf8e85744c446a58839c08df4e109934 Mon Sep 17 00:00:00 2001 From: mattloose Date: Wed, 3 Apr 2024 10:00:18 +0100 Subject: [PATCH 02/16] minor code reformat --- src/readfish/plugins/dorado.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py index f9842de5..3df58987 100644 --- a/src/readfish/plugins/dorado.py +++ b/src/readfish/plugins/dorado.py @@ -66,7 +66,6 @@ def __init__( self.caller = PyBasecallClient(**self.dorado_params) self.caller.connect() - def validate(self) -> None: """Validate the parameters passed to Dorado to ensure they will initialise py Basecall Client correctly From fd3f24cb0a37907ac2566d31fcd78b31bc471023 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 09:53:02 +0100 Subject: [PATCH 03/16] Bump version --- src/readfish/__about__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/readfish/__about__.py b/src/readfish/__about__.py index 4e896e6f..24efc747 100644 --- a/src/readfish/__about__.py +++ b/src/readfish/__about__.py @@ -1,4 +1,5 @@ """__about__.py Version of the read until software """ -__version__ = "2023.1.1" + +__version__ = "2024.0.0" From edb29fb9ea5fdd3faca864c975a19c951e6f553c Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 09:53:19 +0100 Subject: [PATCH 04/16] Update Changelog --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0a0978d..bf9f8650 100644 --- a/README.md +++ b/README.md @@ -554,9 +554,11 @@ And for our Awesome Logo please checkout out [@tim_bassford](https://twitter.com # Changelog ## Unreleased changes -1. Change the default `unblock_duration` on the `Analysis` class to use `DEFAULT_UNBLOCK` value defined in `_cli_args.py`. Change type on the Argparser for `--unblock-duration` to float. (#313) +1. Change the default `unblock_duration` on the `Analysis` class to use `DEFAULT_UNBLOCK` value defined in `_cli_args.py`. Change type on the Argparser for `--unblock-duration` to float. [(#313)](https://github.com/LooseLab/readfish/pull/313) +1. Fix to handling minimap2 indexes with multiple "." in the filename [(#330)](https://github.com/LooseLab/readfish/pull/330/) +1. Add a dorado base-caller which addressed issue [#347](https://github.com/LooseLab/readfish/issues/347) - chiefly in Dorado 7.3.9 ONT have moved to `ont-pybasecall-client-lib`, and connections from `ont_pyguppy_client_lib` raise `Connection error. ... LOAD_CONFIG. Reply: INVALID_PROTOCOL` [(#343)](https://github.com/LooseLab/readfish/pull/344) ## 2023.1.1 1. Fix Readme Logo link 🥳 (#296) -1. Fix bug where we had accidentally started requiring barcoded TOMLs to specify a region. Thanks to @jamesemery for catching this. (#299) -1. Correctly handle overriding a decision in internal statistics tracking. (#299) +2. Fix bug where we had accidentally started requiring barcoded TOMLs to specify a region. Thanks to @jamesemery for catching this. (#299) +3. Correctly handle overriding a decision in internal statistics tracking. (#299) From 5bbe5895afedcf70c2f6a78a44a7c10c15253cc8 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 09:55:10 +0100 Subject: [PATCH 05/16] Add `ont-pybasecall-client-lib` as a dependency under target `dorado` --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6ce56149..41a95dd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,8 @@ dev = ["readfish[all,docs,tests]", "pre-commit"] mappy = ["mappy"] mappy-rs = ["mappy-rs >= 0.0.6"] guppy = ["ont_pyguppy_client_lib"] -all = ["readfish[mappy,mappy-rs,guppy]"] +dorado = ["ont-pybasecall-client-lib"] +all = ["readfish[mappy,mappy-rs,guppy,dorado]"] [project.urls] Documentation = "https://looselab.github.io/readfish" From e591b1fa872d6ac4c11810d0d1452c7e23828b91 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 09:55:35 +0100 Subject: [PATCH 06/16] fix typo --- docs/developers-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 22c9f43f..a32c64ad 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -67,7 +67,7 @@ conda env create -f docs/development.yml ## Readfish versioning Readfish uses [calver](https://calver.org/) for versioning. Specifically the format should be -`YYYY.MINOR.MICRO.Modifier`, where `MINOR` is the feature addiiton, `MICRO` is any hotfix/bugfix, and `Modifier` is the modifier (e.g. `rc` for release candidate, `dev` for development, empty for stable). +`YYYY.MINOR.MICRO.Modifier`, where `MINOR` is the feature addition, `MICRO` is any hotfix/bugfix, and `Modifier` is the modifier (e.g. `rc` for release candidate, `dev` for development, empty for stable). ## Changelog From e0eb6e9343b513a517345867f6eab5fc133658d3 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 11:28:08 +0100 Subject: [PATCH 07/16] Add try/except on import helper functions and pyclient for basecaller This is so pytest collections (The only time both are imported into the same interperter runtime) doesn't crash. This requires all tests to only test one, so I've moved all tests to test dorado --- src/readfish/plugins/dorado.py | 9 +++++++-- src/readfish/plugins/guppy.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py index 3df58987..b0665cd7 100644 --- a/src/readfish/plugins/dorado.py +++ b/src/readfish/plugins/dorado.py @@ -2,6 +2,7 @@ Extension of pyBaseCaller that maintains a connection to the basecaller """ + from __future__ import annotations import logging import os @@ -13,8 +14,12 @@ import numpy as np import numpy.typing as npt from minknow_api.protocol_pb2 import ProtocolRunInfo -from pybasecall_client_lib.helper_functions import package_read -from pybasecall_client_lib.pyclient import PyBasecallClient + +try: + from pybasecall_client_lib.helper_functions import package_read + from pybasecall_client_lib.pyclient import PyBasecallClient +except ImportError: + pass from readfish._loggers import setup_logger from readfish.plugins.abc import CallerABC diff --git a/src/readfish/plugins/guppy.py b/src/readfish/plugins/guppy.py index a2b7d9ec..5cadf1d4 100644 --- a/src/readfish/plugins/guppy.py +++ b/src/readfish/plugins/guppy.py @@ -2,6 +2,7 @@ Extension of pyguppy Caller that maintains a connection to the basecaller """ + from __future__ import annotations import logging import os @@ -13,8 +14,12 @@ import numpy as np import numpy.typing as npt from minknow_api.protocol_pb2 import ProtocolRunInfo -from pyguppy_client_lib.helper_functions import package_read -from pyguppy_client_lib.pyclient import PyGuppyClient + +try: + from pyguppy_client_lib.helper_functions import package_read + from pyguppy_client_lib.pyclient import PyGuppyClient +except ImportError: + pass from readfish._loggers import setup_logger from readfish.plugins.abc import CallerABC From d149c3de9aa2826527103f2caa6949b1654472c0 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 11:31:28 +0100 Subject: [PATCH 08/16] Move tests to test dorado rather than guppy plugin when validating This is due to the fact that pytest imports all files, the only time that pyguppy-client-lib and pybasecall-client-lib are imported at the same time In a real experiment, the plugin choice for base-caller imports the correct module and leaves the other one out of the interpreter runtime --- .../static/guppy_validation_test/fail/001_no_write_socket.toml | 2 +- tests/static/guppy_validation_test/fail/001_no_write_socket.txt | 2 +- tests/static/guppy_validation_test/fail/002_missing_socket.toml | 2 +- tests/static/guppy_validation_test/fail/002_missing_socket.txt | 2 +- tests/static/guppy_validation_test/fail/003_no_read_socket.toml | 2 +- tests/static/guppy_validation_test/fail/003_no_read_socket.txt | 2 +- tests/static/toml_validation_test/fail/001_missing_keys.toml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/static/guppy_validation_test/fail/001_no_write_socket.toml b/tests/static/guppy_validation_test/fail/001_no_write_socket.toml index b33b165c..4cf888dd 100644 --- a/tests/static/guppy_validation_test/fail/001_no_write_socket.toml +++ b/tests/static/guppy_validation_test/fail/001_no_write_socket.toml @@ -1,4 +1,4 @@ -[caller_settings.guppy] +[caller_settings.dorado] address = "ipc://tests/static/guppy_validation_test/fail/5555_fail_nw" config = "dna_r10.4.1_e8.2_400bps_hac" diff --git a/tests/static/guppy_validation_test/fail/001_no_write_socket.txt b/tests/static/guppy_validation_test/fail/001_no_write_socket.txt index 2f5da30b..64365174 100644 --- a/tests/static/guppy_validation_test/fail/001_no_write_socket.txt +++ b/tests/static/guppy_validation_test/fail/001_no_write_socket.txt @@ -1 +1 @@ -The user account running readfish doesn't appear to have permissions to write to the guppy base-caller socket. Please check permissions on ipc://tests/static/guppy_validation_test/fail/5555_fail_nw. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information. +The user account running readfish doesn't appear to have permissions to write to the dorado base-caller socket. Please check permissions on ipc://tests/static/guppy_validation_test/fail/5555_fail_nw. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information. diff --git a/tests/static/guppy_validation_test/fail/002_missing_socket.toml b/tests/static/guppy_validation_test/fail/002_missing_socket.toml index 82dedf57..e21c05f1 100644 --- a/tests/static/guppy_validation_test/fail/002_missing_socket.toml +++ b/tests/static/guppy_validation_test/fail/002_missing_socket.toml @@ -1,4 +1,4 @@ -[caller_settings.guppy] +[caller_settings.dorado] address = "ipc://tests/static/guppy_validation_test/fail/5555_not_there" config = "dna_r10.4.1_e8.2_400bps_hac" diff --git a/tests/static/guppy_validation_test/fail/002_missing_socket.txt b/tests/static/guppy_validation_test/fail/002_missing_socket.txt index 96da59b3..81f5fd7c 100644 --- a/tests/static/guppy_validation_test/fail/002_missing_socket.txt +++ b/tests/static/guppy_validation_test/fail/002_missing_socket.txt @@ -1 +1 @@ -The provided guppy base-caller socket address doesn't appear to exist. Please check your Guppy Settings. ipc://tests/static/guppy_validation_test/fail/5555_not_there +The provided dorado base-caller socket address doesn't appear to exist. Please check your dorado Settings. ipc://tests/static/guppy_validation_test/fail/5555_not_there diff --git a/tests/static/guppy_validation_test/fail/003_no_read_socket.toml b/tests/static/guppy_validation_test/fail/003_no_read_socket.toml index 8670a5cd..c7eaf018 100644 --- a/tests/static/guppy_validation_test/fail/003_no_read_socket.toml +++ b/tests/static/guppy_validation_test/fail/003_no_read_socket.toml @@ -1,4 +1,4 @@ -[caller_settings.guppy] +[caller_settings.dorado] address = "ipc://tests/static/guppy_validation_test/fail/5555_fail_nr" config = "dna_r10.4.1_e8.2_400bps_hac" diff --git a/tests/static/guppy_validation_test/fail/003_no_read_socket.txt b/tests/static/guppy_validation_test/fail/003_no_read_socket.txt index 46973458..5456737d 100644 --- a/tests/static/guppy_validation_test/fail/003_no_read_socket.txt +++ b/tests/static/guppy_validation_test/fail/003_no_read_socket.txt @@ -1 +1 @@ -The user account running readfish doesn't appear to have permissions to read the guppy base-caller socket. Please check permissions on ipc://tests/static/guppy_validation_test/fail/5555_fail_nr. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information. +The user account running readfish doesn't appear to have permissions to read the dorado base-caller socket. Please check permissions on ipc://tests/static/guppy_validation_test/fail/5555_fail_nr. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information. diff --git a/tests/static/toml_validation_test/fail/001_missing_keys.toml b/tests/static/toml_validation_test/fail/001_missing_keys.toml index 7e17a32e..8a5bde1f 100644 --- a/tests/static/toml_validation_test/fail/001_missing_keys.toml +++ b/tests/static/toml_validation_test/fail/001_missing_keys.toml @@ -1,4 +1,4 @@ -[caller_settings.guppy] +[caller_settings.dorado] host = "127.0.0.1" [mapper_settings.mappy] From 6bb34e4427b3e0eed30f22d18ee1154b2efccd12 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 12:04:48 +0100 Subject: [PATCH 09/16] Remove dorado, guppy and _read_until_client from coverage reporting as we can't really cover them They require things link a read_until_api or live base caller to properly test --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 41a95dd8..cfa247fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,3 +84,10 @@ markers = [ "alignment: marks tests which rely on loading or using Mappy or Mappy-rs aligners, used to test with both. (deselect with '-m \"not slow\", select with '-k alignment')", ] addopts = ["-ra", "--doctest-modules", "--ignore=src/readfish/read_until/base.py"] + +[tool.coverage.report] +omit = [ + "src/readfish/plugins/dorado.py", + "src/readfish/_read_until_client.py", + "src/readfish/plugins/guppy.py", +] From a4d8b7e84ddc9f693bf21f893cce5de04e6324aa Mon Sep 17 00:00:00 2001 From: mattloose Date: Sat, 20 Apr 2024 20:39:50 +0100 Subject: [PATCH 10/16] Introducing sample rate from minknow and catching sub_read issue. --- src/readfish/entry_points/stats.py | 1 + src/readfish/entry_points/targets.py | 5 +++- src/readfish/entry_points/unblock_all.py | 1 + src/readfish/plugins/dorado.py | 35 +++++++++++++++++++----- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/readfish/entry_points/stats.py b/src/readfish/entry_points/stats.py index f670c65a..bdbd26df 100644 --- a/src/readfish/entry_points/stats.py +++ b/src/readfish/entry_points/stats.py @@ -73,6 +73,7 @@ readfish stats --toml tests/static/stats_test/yeast_summary_test.toml --fastq-directory tests/static/stats_test/ --html summary_adaptive """ + from __future__ import annotations import argparse from pathlib import Path diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 09da4973..3257be69 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -66,6 +66,7 @@ """ + # Core imports from __future__ import annotations import argparse @@ -171,16 +172,18 @@ def __init__( self.break_reads_after_seconds = ( self.client.connection.analysis_configuration.get_analysis_configuration().read_detection.break_reads_after_seconds.value ) + self.sample_rate = self.client.connection.device.get_sample_rate().sample_rate self.logger.info("Run Configuration Received") self.logger.info(f"run_id={self.run_information.run_id}") self.logger.info(f"break_reads_after_seconds={self.break_reads_after_seconds}") + self.logger.info(f"sample_rate={self.sample_rate}") # Create our statistics tracker self.loop_statistics = ReadfishStatistics( read_log_name, self.break_reads_after_seconds ) logger.info("Initialising Caller") self.caller: CallerABC = self.conf.caller_settings.load_object( - "Caller", run_information=self.run_information + "Caller", run_information=self.run_information, sample_rate=self.sample_rate ) logger.info("Caller initialised") caller_description = self.caller.describe() diff --git a/src/readfish/entry_points/unblock_all.py b/src/readfish/entry_points/unblock_all.py index 8fe1eb74..e40e6292 100644 --- a/src/readfish/entry_points/unblock_all.py +++ b/src/readfish/entry_points/unblock_all.py @@ -12,6 +12,7 @@ readfish unblock-all --device X3 --experiment-name "test unblock all" """ + from tempfile import NamedTemporaryFile from readfish._cli_args import DEVICE_BASE_ARGS diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py index b0665cd7..b3f713e0 100644 --- a/src/readfish/plugins/dorado.py +++ b/src/readfish/plugins/dorado.py @@ -14,6 +14,7 @@ import numpy as np import numpy.typing as npt from minknow_api.protocol_pb2 import ProtocolRunInfo +from minknow_api.device_pb2 import GetSampleRateResponse try: from pybasecall_client_lib.helper_functions import package_read @@ -55,12 +56,20 @@ def __getitem__(self, _): class Caller(CallerABC): def __init__( - self, run_information: ProtocolRunInfo = None, debug_log=None, **kwargs + self, + run_information: ProtocolRunInfo = None, + sample_rate: GetSampleRateResponse = None, + debug_log=None, + **kwargs, ): self.logger = setup_logger("readfish_dorado_logger", log_file=debug_log) self.supported_barcode_kits = None self.supported_basecall_models = None self.run_information = run_information + if sample_rate: + self.sample_rate = float(sample_rate) + else: + self.sample_rate = float(5000) # Set our own priority self.dorado_params = kwargs @@ -198,21 +207,18 @@ def basecall( cache, skipped = {}, {} reads_received, reads_sent = 0, 0 daq_values = _DefaultDAQValues if daq_values is None else daq_values - for channel, read in reads: # Attach the "RF-" prefix read_id = f"RF-{read.id}" t0 = time.time() cache[read_id] = (channel, read.number, t0) - success = self.caller.pass_read( package_read( read_id=read_id, raw_data=np.frombuffer(read.raw_data, signal_dtype), daq_offset=daq_values[channel].offset, daq_scaling=daq_values[channel].scaling, - # ToDo: Sample rate needs to be derived from the MinKNOW API - sampling_rate=float(5000), + sampling_rate=self.sample_rate, start_time=int(read.start_sample), ) ) @@ -237,13 +243,28 @@ def basecall( time.sleep(self.caller.throttle) continue - for res_batch in results: + for res_batch in results: for res in res_batch: read_id = res["metadata"]["read_id"] + ''' + Dorado sometimes returns two sub_tags for a read. + + An example result: + RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c + {'read_tag': 3605, 'sub_tag': 0, 'priority': , 'metadata': {'adapter_front_begin_index': -1, 'adapter_front_foundseq': 'NA', 'adapter_front_foundseq_length': 2, 'adapter_front_id': 'NA', 'adapter_front_refseq': 'NA', 'adapter_front_score': 0.0, 'adapter_mid_end_index': -1, 'adapter_mid_id': 'NA', 'adapter_mid_score': 0.0, 'adapter_rear_end_index': -1, 'adapter_rear_foundseq': 'NA', 'adapter_rear_foundseq_length': 2, 'adapter_rear_id': 'NA', 'adapter_rear_refseq': 'NA', 'adapter_rear_score': 0.0, 'barcode_arrangement': '', 'barcode_front_begin_index': 0, 'barcode_front_foundseq': 'NA', 'barcode_front_foundseq_length': 2, 'barcode_front_id': 'NA', 'barcode_front_id_inner': 'NA', 'barcode_front_refseq': 'NA', 'barcode_front_score': 0.0, 'barcode_front_score_inner': 0.0, 'barcode_full_arrangement': '', 'barcode_kit': '', 'barcode_mid_front_end_index': 0, 'barcode_mid_front_id': 'NA', 'barcode_mid_front_score': 0.0, 'barcode_mid_rear_end_index': 0, 'barcode_mid_rear_id': 'NA', 'barcode_mid_rear_score': 0.0, 'barcode_rear_end_index': 0, 'barcode_rear_foundseq': 'NA', 'barcode_rear_foundseq_length': 2, 'barcode_rear_id': 'NA', 'barcode_rear_id_inner': 'NA', 'barcode_rear_refseq': 'NA', 'barcode_rear_score': 0.0, 'barcode_rear_score_inner': 0.0, 'barcode_score': 0.0, 'barcode_variant': '', 'basecall_type': 'beam search', 'daq_offset': 0.0, 'daq_scaling': 0.25, 'duration': 8592, 'lamp_barcode_id': 'NA', 'lamp_barcode_score': 0.0, 'lamp_target_id': 'NA', 'lamp_target_score': 0.0, 'mean_qscore': 10.297822952270508, 'med_abs_dev': 0.01067463681101799, 'median': -373.5199890136719, 'model_stride': 6, 'model_version_id': 'dna_r10.4.1_e8.2_400bps_fast@v4.3.0', 'num_events': 1432, 'num_minknow_events': 0, 'primer_front_begin_index': -1, 'primer_front_foundseq': 'NA', 'primer_front_foundseq_length': 2, 'primer_front_id': 'NA', 'primer_front_refseq': 'NA', 'primer_front_score': 0.0, 'primer_rear_end_index': -1, 'primer_rear_foundseq': 'NA', 'primer_rear_foundseq_length': 2, 'primer_rear_id': 'NA', 'primer_rear_refseq': 'NA', 'primer_rear_score': 0.0, 'read_id': 'RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c', 'sampling_rate': 5000.0, 'scaling_med_abs_dev': 0.01067463681101799, 'scaling_median': -373.5199890136719, 'scaling_version': '', 'sequence_length': 340, 'start_time': 12462912, 'state_size': 0, 'strand_id': '441174ae-e18f-4612-bc46-0c86d301cb4d', 'trim_front': 0, 'trim_rear': 0, 'trimmed_duration': 8592, 'trimmed_events': 1432, 'trimmed_samples': 0}, 'datasets': {'raw_data': array([306, 289, 300, ..., 526, 537, 540], dtype=int16), 'sequence': 'CTACAGTGGTCGTCAATACTATTGCAACTCCAGCCTGGGCAACAGAGTGAGACCCTGTCTCAAAAAAAAACAGAAGGAAGCCAAAGCCCACAAAGACGCTGAGTATTTTACATTCCTTTTTTGTACGAATGCTTCAAAATCTGGTGTGTATCTTACAGTTATAAAATCTCTCAATTTAGCCACCAAATTTTCACCAGAAATGCATGAACTGTGTTCAGATTTCATAAACTTTCAGTTGATAAAGTAGATTCACATACTGAAGGTGTTCCAAACGTACTGGAAGACTTTCTAAAATGGATTTGAACACCAGTTTCCAGGTTTACATTTAAGTTAATTCAAA', 'qstring': '#(&&%(\'\'%&(#"""$\'%%$%$%*,)+()\'(\'\'*+AEHHH.-0410)200BGC4HKJJGGB77H(\'(+,1-(,//:3640==6504C><357>4448FAFCIGI64-/9660=12HISOBGKLA@?1/59EGM>:::DEJSOM1D?>=GI11775:930,;;>H9/7122)*0:**-===>B55878&#+"#&\',,?@EIJBCLSM964\'(376528:5/-*)21\'##+-&$"#%+&#&%\'-./49?B=A960@7;ABB@6CIEFDD>H7)&45:DC5158B'}} + RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c + {'read_tag': 3605, 'sub_tag': 1, 'priority': , 'metadata': {'adapter_front_begin_index': -1, 'adapter_front_foundseq': 'NA', 'adapter_front_foundseq_length': 2, 'adapter_front_id': 'NA', 'adapter_front_refseq': 'NA', 'adapter_front_score': 0.0, 'adapter_mid_end_index': -1, 'adapter_mid_id': 'NA', 'adapter_mid_score': 0.0, 'adapter_rear_end_index': -1, 'adapter_rear_foundseq': 'NA', 'adapter_rear_foundseq_length': 2, 'adapter_rear_id': 'NA', 'adapter_rear_refseq': 'NA', 'adapter_rear_score': 0.0, 'barcode_arrangement': '', 'barcode_front_begin_index': 0, 'barcode_front_foundseq': 'NA', 'barcode_front_foundseq_length': 2, 'barcode_front_id': 'NA', 'barcode_front_id_inner': 'NA', 'barcode_front_refseq': 'NA', 'barcode_front_score': 0.0, 'barcode_front_score_inner': 0.0, 'barcode_full_arrangement': '', 'barcode_kit': '', 'barcode_mid_front_end_index': 0, 'barcode_mid_front_id': 'NA', 'barcode_mid_front_score': 0.0, 'barcode_mid_rear_end_index': 0, 'barcode_mid_rear_id': 'NA', 'barcode_mid_rear_score': 0.0, 'barcode_rear_end_index': 0, 'barcode_rear_foundseq': 'NA', 'barcode_rear_foundseq_length': 2, 'barcode_rear_id': 'NA', 'barcode_rear_id_inner': 'NA', 'barcode_rear_refseq': 'NA', 'barcode_rear_score': 0.0, 'barcode_rear_score_inner': 0.0, 'barcode_score': 0.0, 'barcode_variant': '', 'basecall_type': 'beam search', 'daq_offset': 0.0, 'daq_scaling': 0.25, 'duration': 321, 'lamp_barcode_id': 'NA', 'lamp_barcode_score': 0.0, 'lamp_target_id': 'NA', 'lamp_target_score': 0.0, 'mean_qscore': 12.145367622375488, 'med_abs_dev': 0.01067463681101799, 'median': -373.5199890136719, 'model_stride': 6, 'model_version_id': 'dna_r10.4.1_e8.2_400bps_fast@v4.3.0', 'num_events': 53, 'num_minknow_events': 0, 'primer_front_begin_index': -1, 'primer_front_foundseq': 'NA', 'primer_front_foundseq_length': 2, 'primer_front_id': 'NA', 'primer_front_refseq': 'NA', 'primer_front_score': 0.0, 'primer_rear_end_index': -1, 'primer_rear_foundseq': 'NA', 'primer_rear_foundseq_length': 2, 'primer_rear_id': 'NA', 'primer_rear_refseq': 'NA', 'primer_rear_score': 0.0, 'read_id': 'RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c', 'sampling_rate': 5000.0, 'scaling_med_abs_dev': 0.01067463681101799, 'scaling_median': -373.5199890136719, 'scaling_version': '', 'sequence_length': 16, 'start_time': 12477610, 'state_size': 0, 'strand_id': '3b536c09-3f73-4c59-a332-2a0f6cad4ed9', 'trim_front': 0, 'trim_rear': 0, 'trimmed_duration': 321, 'trimmed_events': 53, 'trimmed_samples': 0}, 'datasets': {'raw_data': array([306, 289, 300, ..., 526, 537, 540], dtype=int16), 'sequence': 'TTTTTGTGGGCTTTTA', 'qstring': "21;;B98'-213,*2&"}} + + The second sub_tag will cause the code to gail as it is not expected. + + Therefore we need to check for the sub_tag and if it is not 0, we need to skip the read. + ''' + if res["sub_tag"] > 0: + continue + try: channel, read_number, time_sent = cache.pop(read_id) except KeyError: - # FIXME: This is resolved in later versions of dorado. channel, read_number, time_sent = skipped.pop(read_id) reads_sent += 1 res["metadata"]["read_id"] = read_id[3:] From da4431e1b67b701d058a3da026248df4b61ef350 Mon Sep 17 00:00:00 2001 From: mattloose Date: Mon, 22 Apr 2024 17:46:46 +0100 Subject: [PATCH 11/16] Clean trailing whitespace --- src/readfish/plugins/dorado.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py index b3f713e0..34a0acd1 100644 --- a/src/readfish/plugins/dorado.py +++ b/src/readfish/plugins/dorado.py @@ -243,20 +243,20 @@ def basecall( time.sleep(self.caller.throttle) continue - for res_batch in results: + for res_batch in results: for res in res_batch: read_id = res["metadata"]["read_id"] ''' Dorado sometimes returns two sub_tags for a read. - + An example result: RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c {'read_tag': 3605, 'sub_tag': 0, 'priority': , 'metadata': {'adapter_front_begin_index': -1, 'adapter_front_foundseq': 'NA', 'adapter_front_foundseq_length': 2, 'adapter_front_id': 'NA', 'adapter_front_refseq': 'NA', 'adapter_front_score': 0.0, 'adapter_mid_end_index': -1, 'adapter_mid_id': 'NA', 'adapter_mid_score': 0.0, 'adapter_rear_end_index': -1, 'adapter_rear_foundseq': 'NA', 'adapter_rear_foundseq_length': 2, 'adapter_rear_id': 'NA', 'adapter_rear_refseq': 'NA', 'adapter_rear_score': 0.0, 'barcode_arrangement': '', 'barcode_front_begin_index': 0, 'barcode_front_foundseq': 'NA', 'barcode_front_foundseq_length': 2, 'barcode_front_id': 'NA', 'barcode_front_id_inner': 'NA', 'barcode_front_refseq': 'NA', 'barcode_front_score': 0.0, 'barcode_front_score_inner': 0.0, 'barcode_full_arrangement': '', 'barcode_kit': '', 'barcode_mid_front_end_index': 0, 'barcode_mid_front_id': 'NA', 'barcode_mid_front_score': 0.0, 'barcode_mid_rear_end_index': 0, 'barcode_mid_rear_id': 'NA', 'barcode_mid_rear_score': 0.0, 'barcode_rear_end_index': 0, 'barcode_rear_foundseq': 'NA', 'barcode_rear_foundseq_length': 2, 'barcode_rear_id': 'NA', 'barcode_rear_id_inner': 'NA', 'barcode_rear_refseq': 'NA', 'barcode_rear_score': 0.0, 'barcode_rear_score_inner': 0.0, 'barcode_score': 0.0, 'barcode_variant': '', 'basecall_type': 'beam search', 'daq_offset': 0.0, 'daq_scaling': 0.25, 'duration': 8592, 'lamp_barcode_id': 'NA', 'lamp_barcode_score': 0.0, 'lamp_target_id': 'NA', 'lamp_target_score': 0.0, 'mean_qscore': 10.297822952270508, 'med_abs_dev': 0.01067463681101799, 'median': -373.5199890136719, 'model_stride': 6, 'model_version_id': 'dna_r10.4.1_e8.2_400bps_fast@v4.3.0', 'num_events': 1432, 'num_minknow_events': 0, 'primer_front_begin_index': -1, 'primer_front_foundseq': 'NA', 'primer_front_foundseq_length': 2, 'primer_front_id': 'NA', 'primer_front_refseq': 'NA', 'primer_front_score': 0.0, 'primer_rear_end_index': -1, 'primer_rear_foundseq': 'NA', 'primer_rear_foundseq_length': 2, 'primer_rear_id': 'NA', 'primer_rear_refseq': 'NA', 'primer_rear_score': 0.0, 'read_id': 'RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c', 'sampling_rate': 5000.0, 'scaling_med_abs_dev': 0.01067463681101799, 'scaling_median': -373.5199890136719, 'scaling_version': '', 'sequence_length': 340, 'start_time': 12462912, 'state_size': 0, 'strand_id': '441174ae-e18f-4612-bc46-0c86d301cb4d', 'trim_front': 0, 'trim_rear': 0, 'trimmed_duration': 8592, 'trimmed_events': 1432, 'trimmed_samples': 0}, 'datasets': {'raw_data': array([306, 289, 300, ..., 526, 537, 540], dtype=int16), 'sequence': 'CTACAGTGGTCGTCAATACTATTGCAACTCCAGCCTGGGCAACAGAGTGAGACCCTGTCTCAAAAAAAAACAGAAGGAAGCCAAAGCCCACAAAGACGCTGAGTATTTTACATTCCTTTTTTGTACGAATGCTTCAAAATCTGGTGTGTATCTTACAGTTATAAAATCTCTCAATTTAGCCACCAAATTTTCACCAGAAATGCATGAACTGTGTTCAGATTTCATAAACTTTCAGTTGATAAAGTAGATTCACATACTGAAGGTGTTCCAAACGTACTGGAAGACTTTCTAAAATGGATTTGAACACCAGTTTCCAGGTTTACATTTAAGTTAATTCAAA', 'qstring': '#(&&%(\'\'%&(#"""$\'%%$%$%*,)+()\'(\'\'*+AEHHH.-0410)200BGC4HKJJGGB77H(\'(+,1-(,//:3640==6504C><357>4448FAFCIGI64-/9660=12HISOBGKLA@?1/59EGM>:::DEJSOM1D?>=GI11775:930,;;>H9/7122)*0:**-===>B55878&#+"#&\',,?@EIJBCLSM964\'(376528:5/-*)21\'##+-&$"#%+&#&%\'-./49?B=A960@7;ABB@6CIEFDD>H7)&45:DC5158B'}} RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c {'read_tag': 3605, 'sub_tag': 1, 'priority': , 'metadata': {'adapter_front_begin_index': -1, 'adapter_front_foundseq': 'NA', 'adapter_front_foundseq_length': 2, 'adapter_front_id': 'NA', 'adapter_front_refseq': 'NA', 'adapter_front_score': 0.0, 'adapter_mid_end_index': -1, 'adapter_mid_id': 'NA', 'adapter_mid_score': 0.0, 'adapter_rear_end_index': -1, 'adapter_rear_foundseq': 'NA', 'adapter_rear_foundseq_length': 2, 'adapter_rear_id': 'NA', 'adapter_rear_refseq': 'NA', 'adapter_rear_score': 0.0, 'barcode_arrangement': '', 'barcode_front_begin_index': 0, 'barcode_front_foundseq': 'NA', 'barcode_front_foundseq_length': 2, 'barcode_front_id': 'NA', 'barcode_front_id_inner': 'NA', 'barcode_front_refseq': 'NA', 'barcode_front_score': 0.0, 'barcode_front_score_inner': 0.0, 'barcode_full_arrangement': '', 'barcode_kit': '', 'barcode_mid_front_end_index': 0, 'barcode_mid_front_id': 'NA', 'barcode_mid_front_score': 0.0, 'barcode_mid_rear_end_index': 0, 'barcode_mid_rear_id': 'NA', 'barcode_mid_rear_score': 0.0, 'barcode_rear_end_index': 0, 'barcode_rear_foundseq': 'NA', 'barcode_rear_foundseq_length': 2, 'barcode_rear_id': 'NA', 'barcode_rear_id_inner': 'NA', 'barcode_rear_refseq': 'NA', 'barcode_rear_score': 0.0, 'barcode_rear_score_inner': 0.0, 'barcode_score': 0.0, 'barcode_variant': '', 'basecall_type': 'beam search', 'daq_offset': 0.0, 'daq_scaling': 0.25, 'duration': 321, 'lamp_barcode_id': 'NA', 'lamp_barcode_score': 0.0, 'lamp_target_id': 'NA', 'lamp_target_score': 0.0, 'mean_qscore': 12.145367622375488, 'med_abs_dev': 0.01067463681101799, 'median': -373.5199890136719, 'model_stride': 6, 'model_version_id': 'dna_r10.4.1_e8.2_400bps_fast@v4.3.0', 'num_events': 53, 'num_minknow_events': 0, 'primer_front_begin_index': -1, 'primer_front_foundseq': 'NA', 'primer_front_foundseq_length': 2, 'primer_front_id': 'NA', 'primer_front_refseq': 'NA', 'primer_front_score': 0.0, 'primer_rear_end_index': -1, 'primer_rear_foundseq': 'NA', 'primer_rear_foundseq_length': 2, 'primer_rear_id': 'NA', 'primer_rear_refseq': 'NA', 'primer_rear_score': 0.0, 'read_id': 'RF-17774e44-8d3c-4bdd-8730-aaf8addd3e6c', 'sampling_rate': 5000.0, 'scaling_med_abs_dev': 0.01067463681101799, 'scaling_median': -373.5199890136719, 'scaling_version': '', 'sequence_length': 16, 'start_time': 12477610, 'state_size': 0, 'strand_id': '3b536c09-3f73-4c59-a332-2a0f6cad4ed9', 'trim_front': 0, 'trim_rear': 0, 'trimmed_duration': 321, 'trimmed_events': 53, 'trimmed_samples': 0}, 'datasets': {'raw_data': array([306, 289, 300, ..., 526, 537, 540], dtype=int16), 'sequence': 'TTTTTGTGGGCTTTTA', 'qstring': "21;;B98'-213,*2&"}} - + The second sub_tag will cause the code to gail as it is not expected. - + Therefore we need to check for the sub_tag and if it is not 0, we need to skip the read. ''' if res["sub_tag"] > 0: From 6af5ff80c5a2a275bfbec3418153ee87e500d46d Mon Sep 17 00:00:00 2001 From: mattloose Date: Wed, 24 Apr 2024 11:01:25 +0100 Subject: [PATCH 12/16] Updating documentation for dorado --- README.md | 16 +++++++++++----- docs/_static/example_tomls/barcoded_human.toml | 6 ++++-- .../example_tomls/human_bed_file_selection.toml | 6 ++++-- .../example_tomls/human_chr_depletion.toml | 6 ++++-- .../example_tomls/human_chr_selection.toml | 6 ++++-- .../human_chr_selection_two_regions.toml | 6 ++++-- .../human_chr_selection_with_control.toml | 6 ++++-- .../example_tomls/human_csv_file_selection.toml | 10 +++++++--- .../human_minimap2_extra_params.toml | 6 ++++-- .../example_tomls/human_regions_barcoded.toml | 6 ++++-- 10 files changed, 50 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index bf9f8650..103249b0 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ the read in progress and so direct sequencing capacity towards reads of interest **This implementation of readfish requires Guppy version >= 6.0.0 and MinKNOW version core >= 5.0.0 . It will not work on earlier versions.** +**Since MinKNOW version core >=5.9.0 and Dorado server version >=7.3.9, Dorado requires an alternate library, `ont-pybasecall-client-lib`. We have introduced a new`dorado` module to handle this.** + The code here has been tested with Guppy in GPU mode using GridION Mk1 and NVIDIA RTX2080 on live sequencing runs and an NVIDIA GTX1080 using playback @@ -121,9 +123,9 @@ conda env create -f development.yml conda activate readfish_dev ``` -|

‼️ Important!

| -|:---------------------------| -| The listed `ont-pyguppy-client-lib` version will probably not match the version installed on your system. To fix this, Please see this [issue](https://github.com/LooseLab/readfish/issues/221#issuecomment-1381529409) | +|

‼️ Important !!

| +|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| MinKNOW is transitioning from Guppy to Dorado. Until MinKNOW version 5.9 both Guppy and Dorado used ont-pyguppy-client-lib.
As of MinKNOW version 5.9 and Dorado server version 7.3.9 and greater Dorado requires an alternate library, `ont-pybasecall-client-lib`.
The listed `ont-pyguppy-client-lib` or `ont-pybasecaller-client-lib` version may not match the version installed on your system. To fix this, Please see this [issue](https://github.com/LooseLab/readfish/issues/221#issuecomment-1381529409), using the appropriate library. | [ONT's Guppy GPU](https://community.nanoporetech.com/downloads) should be installed and running as a server. @@ -333,8 +335,8 @@ Note: The plots here are generated from running readfish unblock-all on an Apple

Testing base-calling and mapping

 -To test selective sequencing you must have access to a -[guppy basecall server](https://community.nanoporetech.com/downloads/guppy/release_notes) (>=6.0.0) +To test selective sequencing you must have access to either a +[guppy basecall server](https://community.nanoporetech.com/downloads/guppy/release_notes) (>=6.0.0) or a [dorado basecall server](https://community.nanoporetech.com/downloads/dorado/release_notes). and a readfish TOML configuration file. @@ -348,6 +350,10 @@ NOTE: guppy and dorado are used here interchangeably as the basecall server. Dor ```toml [mapper_settings.mappy-rs] ``` +1. If on MinKNOW core>=5.9.0 and Dorado server version >=7.3.9, edit the `basecaller` section to read: + ```toml + [caller_settings.dorado] + ``` 1. Modify the `fn_idx_in` field in the file to be the full path to a [minimap2](https://github.com/lh3/minimap2) index of the human genome. 1. Modify the `targets` fields for each condition to reflect the naming convention used in your index. This is the sequence name only, up to but not including any whitespace. diff --git a/docs/_static/example_tomls/barcoded_human.toml b/docs/_static/example_tomls/barcoded_human.toml index 014baee0..20c5d568 100644 --- a/docs/_static/example_tomls/barcoded_human.toml +++ b/docs/_static/example_tomls/barcoded_human.toml @@ -22,9 +22,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r10.4.1_e8.2_400bps_5khz_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_bed_file_selection.toml b/docs/_static/example_tomls/human_bed_file_selection.toml index 282af9ad..ebeb3263 100644 --- a/docs/_static/example_tomls/human_bed_file_selection.toml +++ b/docs/_static/example_tomls/human_bed_file_selection.toml @@ -14,9 +14,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r9.4.1_450bps_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_chr_depletion.toml b/docs/_static/example_tomls/human_chr_depletion.toml index a241e827..48d78342 100644 --- a/docs/_static/example_tomls/human_chr_depletion.toml +++ b/docs/_static/example_tomls/human_chr_depletion.toml @@ -23,9 +23,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r9.4.1_450bps_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_chr_selection.toml b/docs/_static/example_tomls/human_chr_selection.toml index f522b57d..96bc2dab 100644 --- a/docs/_static/example_tomls/human_chr_selection.toml +++ b/docs/_static/example_tomls/human_chr_selection.toml @@ -18,9 +18,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r9.4.1_450bps_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_chr_selection_two_regions.toml b/docs/_static/example_tomls/human_chr_selection_two_regions.toml index 09ebd6b9..78a6c1cb 100644 --- a/docs/_static/example_tomls/human_chr_selection_two_regions.toml +++ b/docs/_static/example_tomls/human_chr_selection_two_regions.toml @@ -19,9 +19,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r9.4.1_450bps_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_chr_selection_with_control.toml b/docs/_static/example_tomls/human_chr_selection_with_control.toml index ad797fcd..40fd8353 100644 --- a/docs/_static/example_tomls/human_chr_selection_with_control.toml +++ b/docs/_static/example_tomls/human_chr_selection_with_control.toml @@ -21,9 +21,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r9.4.1_450bps_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_csv_file_selection.toml b/docs/_static/example_tomls/human_csv_file_selection.toml index 2484d407..11da24fb 100644 --- a/docs/_static/example_tomls/human_csv_file_selection.toml +++ b/docs/_static/example_tomls/human_csv_file_selection.toml @@ -17,9 +17,13 @@ # - https://looselab.github.io/readfish/toml.html. [caller_settings.guppy] -# Caller Configuration for Guppy Basecaller -config = "dna_r9.4.1_450bps_fast" # Specify the basecaller configuration -address = "ipc:///tmp/.guppy/5555" # Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# ^^^^^^ - ".guppy" specifies our chosen basecaller +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name +config = "dna_r9.4.1_450bps_fast" +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +address = "ipc:///tmp/.guppy/5555" debug_log = "live_reads.fq" # Fastq output for individual reads (Optional, delete line to disable) [mapper_settings.mappy] diff --git a/docs/_static/example_tomls/human_minimap2_extra_params.toml b/docs/_static/example_tomls/human_minimap2_extra_params.toml index b495cbef..83ab44bc 100644 --- a/docs/_static/example_tomls/human_minimap2_extra_params.toml +++ b/docs/_static/example_tomls/human_minimap2_extra_params.toml @@ -23,9 +23,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r10.4.1_e8.2_400bps_5khz_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. diff --git a/docs/_static/example_tomls/human_regions_barcoded.toml b/docs/_static/example_tomls/human_regions_barcoded.toml index 7a0e7556..11d65b43 100644 --- a/docs/_static/example_tomls/human_regions_barcoded.toml +++ b/docs/_static/example_tomls/human_regions_barcoded.toml @@ -22,9 +22,11 @@ # Basecaller configuration [caller_settings.guppy] # ^^^^^^ - ".guppy" specifies our chosen basecaller -# Guppy base-calling configuration file name +# If using dorado >7.3.9, this should be ".dorado". +# All other parameters are shared between the two basecallers. +# Guppy/Dorado base-calling configuration file name config = "dna_r10.4.1_e8.2_400bps_5khz_fast" -# Address of the guppy basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. +# Address of the guppy/dorado basecaller - The default address for guppy is ipc:///tmp/.guppy/5555. address = "ipc:///tmp/.guppy/5555" # Fastq output for individual reads. This is OPTIONAL - as these files can become quite large. # Remove line to disable. From 3482d59403da8290610e698c4cde51a74507a2f0 Mon Sep 17 00:00:00 2001 From: mattloose Date: Tue, 7 May 2024 17:58:51 +0100 Subject: [PATCH 13/16] Feature/check minknow (#351) * Initial validation of minknow version and guppy dorado versions * MinKNOW compatibility check function Base-caller compatibiity check * Compatibility checks * Remove rogue comment * edit insanely long string multiline string example of sub tags * Local updates * Error checking and minor corrections to address compatibility. We no longer raise RuntimeException when a version issue is encountered. This means that readfish will continue to function with future versions of minKNOW if compatible. * Add default when popping `sample_rate` from guppy params Prevents `KeyError` from being raised if `sample_rate`isn't listed as a `kwarg` * Remove unused basecaller compatibilty function We now just check in `dorado.py` and warn if there is a mismatch * Correct docstring, add doctests * Add more complete docstring to `DIRECTION` enum Only return Enum variant from `check_compatibility`, not tuple of (bool, variant) * Suggest opening an issue if a suitable version of readfish doesn't exist --------- Co-authored-by: Adoni5 --- src/readfish/_compatibility.py | 100 ++++++++++++++++++++++++++ src/readfish/_utils.py | 1 + src/readfish/entry_points/targets.py | 28 +++++++- src/readfish/entry_points/validate.py | 9 ++- src/readfish/plugins/dorado.py | 27 ++++--- src/readfish/plugins/guppy.py | 16 +++++ 6 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 src/readfish/_compatibility.py diff --git a/src/readfish/_compatibility.py b/src/readfish/_compatibility.py new file mode 100644 index 00000000..e7e30ba2 --- /dev/null +++ b/src/readfish/_compatibility.py @@ -0,0 +1,100 @@ +"""_compatibility.py + +Contains utilities for checking readfish compatibility with various versions of MinKNOW. + +Checks ranges of `readfish` against the `MinKNOW` version + +Attributes: + LATEST_TESTED (str): The latest tested version of MinKNOW. + MINKNOW_COMPATIBILITY_RANGE (tuple): The compatibility range of MinKNOW versions for this version of readfish. + DIRECTION (Enum): An enumeration representing upgrade, downgrade, or no change directions. + +""" + +from minknow_api.manager import Manager +from packaging.version import parse as parse_version +from packaging.version import Version +from enum import Enum + +LATEST_TESTED = "5.9.7" + +# The versions of MinKNOW which this version of readfish can connect to +# Format - (lowest minknow version, highest version of minknow supported as an upper bound) +MINKNOW_COMPATIBILITY_RANGE = ( + Version("5.0.0"), + Version(LATEST_TESTED), +) + + +class DIRECTION(Enum): + """ + Represents the direction in which the version of the readfish software should be changed + to be compatible with the tested version of an external tool (likely MinKNOW). + + Attributes: + UPGRADE: Indicates that the readfish software version should be upgraded. + DOWNGRADE: Indicates that the readfish software version should be downgraded. + JUST_RIGHT: Indicates that the readfish software version is already compatible + with the tested version of the external tool. + """ + + UPGRADE = "upgrade" + DOWNGRADE = "downgrade" + JUST_RIGHT = "do nothing" + + +def _get_minknow_version(host: str = "127.0.0.1", port: int = None) -> Version: + """ + Get the version of MinKNOW + + :param host: The host the RPC is listening on, defaults to "127.0.0.1" + :param port: The port the RPC is listening on, defaults to None + + :return: The version of MinKNOW readfish is connected to + """ + manager = Manager(host=host, port=port) + minknow_version = parse_version(manager.core_version) + return minknow_version + + +def check_compatibility( + comparator: Version, + version_range: tuple[Version, Version] = MINKNOW_COMPATIBILITY_RANGE, +) -> DIRECTION: + """ + Check the compatibility of a given software version, between a given range, + inclusive of the right edge. + + :param comparator: Version of the provided software, for example MinKNOW 5.9.7 + :param version_ranges: A tuple of lowest supported version, highest supported version + + :return: A direction variant indicating if this version of readfish needs to be changed. + + Examples: + >>> from packaging.version import Version + >>> check_compatibility(Version("5.9.5"), (Version("5.0.0"), Version("5.9.7"))) + + >>> check_compatibility(Version("5.9.7"), (Version("5.0.0"), Version("5.9.7"))) + + >>> check_compatibility(Version("5.9.8"), (Version("5.0.0"), Version("5.9.7"))) + + >>> check_compatibility(Version("4.9.0"), (Version("5.0.0"), Version("5.9.7"))) + + >>> if (action := check_compatibility(Version("6.0.0"), MINKNOW_COMPATIBILITY_RANGE)) in ( + ... DIRECTION.UPGRADE, + ... DIRECTION.DOWNGRADE, + ... ): + ... action + + """ + ( + lowest_supported_version, + highest_supported_version, + ) = version_range + if comparator < lowest_supported_version: + return DIRECTION.DOWNGRADE + return ( + DIRECTION.JUST_RIGHT + if comparator <= highest_supported_version + else DIRECTION.UPGRADE + ) diff --git a/src/readfish/_utils.py b/src/readfish/_utils.py index 991ac160..458584e9 100644 --- a/src/readfish/_utils.py +++ b/src/readfish/_utils.py @@ -18,6 +18,7 @@ from minknow_api.manager import Manager, FlowCellPosition from minknow_api import Connection + if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 3257be69..5ea96c3d 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -86,6 +86,13 @@ from readfish._read_until_client import RUClient from readfish._config import Action, Conf, make_decision, _Condition from readfish._statistics import ReadfishStatistics +from readfish.__about__ import __version__ +from readfish._compatibility import ( + _get_minknow_version, + check_compatibility, + MINKNOW_COMPATIBILITY_RANGE, + DIRECTION, +) from readfish._utils import ( get_device, send_message, @@ -485,6 +492,24 @@ def run( # Setup logger used in this entry point, this one should be passed through logger = logging.getLogger(f"readfish.{args.command}") + # Check MinKNOW version + + minknow_version = _get_minknow_version(host=args.host, port=args.port) + if ( + action := check_compatibility(minknow_version, MINKNOW_COMPATIBILITY_RANGE) + ) in ( + DIRECTION.UPGRADE, + DIRECTION.DOWNGRADE, + ): + lower_bound, upper_bound = MINKNOW_COMPATIBILITY_RANGE + logger.warning( + f"""This readfish version ({__version__}) is tested for compatibility with MinKNOW v{lower_bound} to v{upper_bound}. +This version of minknow is {minknow_version}. +If readfish fails please try to {action.value} readfish. +If there isn't a newer version of readfish and readfish is failing, please open an issue: + https://github.com/LooseLab/readfish/issues""" + ) + # Fetch sequencing device position = get_device(args.device, host=args.host, port=args.port) @@ -516,9 +541,6 @@ def run( # start the client running read_until_client.run( - # TODO: Set correct channel range - # first_channel=186, - # last_channel=187, first_channel=1, last_channel=read_until_client.channel_count, max_unblock_read_length_seconds=args.max_unblock_read_length_seconds, diff --git a/src/readfish/entry_points/validate.py b/src/readfish/entry_points/validate.py index f187703e..f98cc0ea 100644 --- a/src/readfish/entry_points/validate.py +++ b/src/readfish/entry_points/validate.py @@ -81,7 +81,14 @@ def run(parser, args, extras) -> int: try: _ = conf.caller_settings.load_object("Caller") except Exception as exc: - logger.error("Caller could not be initialised, see below for details") + logger.error("Caller could not be initialised.") + logger.error( + "Possible reasons for this include a mismatch between the ont basecaller client and the versions of guppy or dorado you are connecting to." + ) + logger.error( + "Additional information is available here: https://looselab.github.io/readfish/FAQ.html#connection-error-bad-reply-could-not-interpret-message-from-server-for-request-load-config-reply-invalid-protocol" + ) + logger.error("See below for further details on this specific error.") logger.error(str(exc)) errors += 1 else: diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py index 34a0acd1..aa4f6802 100644 --- a/src/readfish/plugins/dorado.py +++ b/src/readfish/plugins/dorado.py @@ -10,6 +10,7 @@ from collections import namedtuple from pathlib import Path from typing import Iterable, TYPE_CHECKING +from packaging.version import parse as parse_version import numpy as np import numpy.typing as npt @@ -66,16 +67,29 @@ def __init__( self.supported_barcode_kits = None self.supported_basecall_models = None self.run_information = run_information - if sample_rate: - self.sample_rate = float(sample_rate) - else: - self.sample_rate = float(5000) + if self.run_information: + self.guppy_version = ( + self.run_information.software_versions.guppy_connected_version + ) + + if parse_version(self.guppy_version) >= parse_version("7.3.9"): + logging.info(f"Connected to caller version {self.guppy_version}.") + else: + logging.info( + f"Trying to use minKNOW with a caller version {self.guppy_version}. If this is causing readfish to crash, try using a version of Dorado >= 7.3.9. You should also check for any updates available to readfish." + ) # Set our own priority self.dorado_params = kwargs self.dorado_params["priority"] = PyBasecallClient.high_priority # Set our own client name to appear in the dorado server logs self.dorado_params["client_name"] = "Readfish_connection" + + if sample_rate: + self.sample_rate = float(sample_rate) + else: + self.sample_rate = float(5000) + self.validate() self.caller = PyBasecallClient(**self.dorado_params) self.caller.connect() @@ -109,10 +123,7 @@ def validate(self) -> None: raise RuntimeError( f"The user account running readfish doesn't appear to have permissions to read the dorado base-caller socket. Please check permissions on {self.dorado_params['address']}. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information." ) - if not os.access(socket_path, os.W_OK): - raise RuntimeError( - f"The user account running readfish doesn't appear to have permissions to write to the dorado base-caller socket. Please check permissions on {self.dorado_params['address']}. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information." - ) + # If we are connected to a live run, test if the base-caller model is acceptable. # Connected to a live run via the minknow_api - get supported basecall and barcoding kits from the run info. # Check them against provided values diff --git a/src/readfish/plugins/guppy.py b/src/readfish/plugins/guppy.py index 5cadf1d4..de9f5fde 100644 --- a/src/readfish/plugins/guppy.py +++ b/src/readfish/plugins/guppy.py @@ -10,6 +10,7 @@ from collections import namedtuple from pathlib import Path from typing import Iterable, TYPE_CHECKING +from packaging.version import parse as parse_version import numpy as np import numpy.typing as npt @@ -62,8 +63,23 @@ def __init__( self.supported_basecall_models = None self.run_information = run_information + if self.run_information: + self.guppy_version = ( + self.run_information.software_versions.guppy_connected_version + ) + + if parse_version(self.guppy_version) < parse_version("7.3.9"): + logging.info(f"Connected to caller version {self.guppy_version}.") + else: + logging.info( + f"Trying to connect to minKNOW with caller version {self.guppy_version}. This plugin requires a version of Dorado or Guppy < 7.3.9. If this is stopping readfish from running try changing [caller_settings.guppy] to [caller_settings.dorado]. You should also check for any updates available to readfish." + ) + # Set our own priority self.guppy_params = kwargs + + # Remove the sample rate from the guppy params as it isn't required for the PyGuppyClient + self.guppy_params.pop("sample_rate", None) self.guppy_params["priority"] = PyGuppyClient.high_priority # Set our own client name to appear in the guppy server logs self.guppy_params["client_name"] = "Readfish_connection" From 19d40796c41ccba43c3342b63dfb7072c651b084 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 9 May 2024 14:36:47 +0100 Subject: [PATCH 14/16] Add __futures__ annotation for Py38 compatibility --- src/readfish/_compatibility.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/readfish/_compatibility.py b/src/readfish/_compatibility.py index e7e30ba2..8a44a47f 100644 --- a/src/readfish/_compatibility.py +++ b/src/readfish/_compatibility.py @@ -11,10 +11,13 @@ """ +from __future__ import annotations + +from enum import Enum + from minknow_api.manager import Manager from packaging.version import parse as parse_version from packaging.version import Version -from enum import Enum LATEST_TESTED = "5.9.7" From d38d5abcb61e46e38fd4ea2585949b389471012b Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 9 May 2024 14:46:08 +0100 Subject: [PATCH 15/16] Deprecation warning on Guppy plugin --- src/readfish/plugins/guppy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/readfish/plugins/guppy.py b/src/readfish/plugins/guppy.py index de9f5fde..4432cc0d 100644 --- a/src/readfish/plugins/guppy.py +++ b/src/readfish/plugins/guppy.py @@ -62,7 +62,9 @@ def __init__( self.supported_barcode_kits = None self.supported_basecall_models = None self.run_information = run_information - + logging.warn( + "Deprecation warning - As ONT has moved fully to dorado, this plugin is no longer maintained, and will be deprecated in a future release of readfish." + ) if self.run_information: self.guppy_version = ( self.run_information.software_versions.guppy_connected_version From 4bf46b4a4e31900a165518b6ff07e2fe052f4518 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 9 May 2024 15:15:47 +0100 Subject: [PATCH 16/16] Add check for Write permission on Dorado socket --- src/readfish/plugins/dorado.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/readfish/plugins/dorado.py b/src/readfish/plugins/dorado.py index aa4f6802..b07e0f3f 100644 --- a/src/readfish/plugins/dorado.py +++ b/src/readfish/plugins/dorado.py @@ -123,7 +123,10 @@ def validate(self) -> None: raise RuntimeError( f"The user account running readfish doesn't appear to have permissions to read the dorado base-caller socket. Please check permissions on {self.dorado_params['address']}. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information." ) - + if not os.access(socket_path, os.W_OK): + raise RuntimeError( + f"The user account running readfish doesn't appear to have permissions to write to the dorado base-caller socket. Please check permissions on {self.dorado_params['address']}. See https://github.com/LooseLab/readfish/issues/221#issuecomment-1375673490 for more information." + ) # If we are connected to a live run, test if the base-caller model is acceptable. # Connected to a live run via the minknow_api - get supported basecall and barcoding kits from the run info. # Check them against provided values