From 246aa63d6a9202b429d1bd6960ace0af734f541f Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Tue, 23 Jan 2024 15:25:37 +0000 Subject: [PATCH 01/33] Remove default params from analysis cli --- src/readfish/entry_points/targets.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 09da4973..4a727aff 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -81,7 +81,7 @@ from minknow_api import protocol_service # Library -from readfish._cli_args import DEVICE_BASE_ARGS, DEFAULT_UNBLOCK +from readfish._cli_args import DEVICE_BASE_ARGS from readfish._read_until_client import RUClient from readfish._config import Action, Conf, make_decision, _Condition from readfish._statistics import ReadfishStatistics @@ -147,10 +147,10 @@ def __init__( conf: Conf, logger: logging.Logger, debug_log: bool, - throttle: float = 0.1, - unblock_duration: float = DEFAULT_UNBLOCK, - dry_run: bool = False, - toml: str = "a.toml", + throttle: float, + unblock_duration: float, + dry_run: bool, + toml: str, ): self.client = client self.conf = conf From 55607a9ea73077ed8c1dfe4d74b29128574cc674 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Tue, 23 Jan 2024 15:25:53 +0000 Subject: [PATCH 02/33] Add duplex CLI Arg --- src/readfish/_cli_args.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/readfish/_cli_args.py b/src/readfish/_cli_args.py index 2102afa0..521f3c46 100644 --- a/src/readfish/_cli_args.py +++ b/src/readfish/_cli_args.py @@ -133,4 +133,12 @@ type=int, ), ), + ( + "--duplex", + dict( + help="**EXPERIMENTAL** Enable duplex targets mode. Accepts reads that align to the opposite strand and same contig as the previous read on channel.", + required=False, + action="store_true", + ), + ), ) + BASE_ARGS From 6358fbe725748b0250f8a2fdb746202f1c9903f6 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 24 Jan 2024 21:15:53 +0000 Subject: [PATCH 03/33] Add duplex override code --- src/readfish/entry_points/targets.py | 53 ++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 4a727aff..e9dd2e55 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -81,7 +81,7 @@ from minknow_api import protocol_service # Library -from readfish._cli_args import DEVICE_BASE_ARGS +from readfish._cli_args import DEVICE_BASE_ARGS, Chemistry from readfish._read_until_client import RUClient from readfish._config import Action, Conf, make_decision, _Condition from readfish._statistics import ReadfishStatistics @@ -92,7 +92,12 @@ Severity, ) from readfish.plugins.abc import AlignerABC, CallerABC -from readfish.plugins.utils import Decision, PreviouslySentActionTracker, Result +from readfish.plugins.utils import ( + Decision, + PreviouslySentActionTracker, + Result, + DuplexTracker, +) _help = "Run targeted sequencing" @@ -151,6 +156,7 @@ def __init__( unblock_duration: float, dry_run: bool, toml: str, + chemistry: Chemistry, ): self.client = client self.conf = conf @@ -194,6 +200,8 @@ def __init__( # This is an object to keep track of the last action sent to the client for each channel self.previous_action_tracker = PreviouslySentActionTracker() + # Keep track of previous alignments + self.duplex_tracker = DuplexTracker() # We assume that sequencing is already running. # If the run is not in sequencing phase when the read until loop starts will @@ -306,18 +314,53 @@ def check_override_action( # previous_action will be None if the read has not been seen before. previous_action = self.previous_action_tracker.get_action(result.channel) action_overridden = False + # If --duplex flag override decisions made based on the strand and contig alignment of the previous read. + if ( + self.chemistry == Chemistry.DUPLEX + and any( + self.previous_alignment_tracker.possible_duplex( + result.channel, result.read_id, al.ctg, al.strand + ) + for al in result.alignment_data + ) + # Previous action was to sequence + and previous_action == Action.stop_receiving + # And we aren't already sequencing it + and action != Action.stop_receiving + and self.duplex_tracker.get_previous_decision() != Decision.duplex_override + ): + self.logger.debug( + f"Overriding read {result.read_id} as it is possibly second half of a duplex" + ) + action_overridden = True + result.decision = Decision.duplex_on + action = Action.stop_receiving + # Duplex + elif ( + self.chemistry == Chemistry.DUPLEX_SIMPLE + and previous_action == Action.stop_receiving + and action != Action.stop_receiving + and self.duplex_tracker.get_previous_decision() != Decision.duplex_override + ): + action = Action.stop_receiving + action_overridden = True + result.decision = Decision.duplex_override + # Override to stop receiving if this is the first read ona channel and we started mid sequencing if previous_action is None and self.readfish_started_during_sequencing: self.logger.debug( f"This is the first suitable read chunk from channel {result.channel}. Translocated read length unknown, sequencing." ) action_overridden = True + result.decision = Decision.first_read_override action = Action.stop_receiving if action is Action.stop_receiving: stop_receiving_action_list.append((result.channel, result.read_number)) # Add decided Action self.previous_action_tracker.add_action(result.channel, action) + # Add final decision - used to check if it is a duplex override + self.duplex_tracker.set_previous_decision(result.decision) elif action is Action.unblock: if self.dry_run: # Log an 'unblock' action to previous action, but send a 'stop receiving' to prevent further read processing. @@ -329,6 +372,8 @@ def check_override_action( ) # Add decided Action self.previous_action_tracker.add_action(result.channel, action) + self.duplex_tracker.set_previous_decision(result.decision) + return ( previous_action, action_overridden, @@ -436,6 +481,9 @@ def run(self): ), overridden_action_name=overridden_action_name, ) + self.previous_alignment_tracker.add_alignment( + result.channel, + ) ####################################################################### # Compile actions to be sent @@ -530,6 +578,7 @@ def run( throttle=args.throttle, dry_run=args.dry_run, toml=args.toml, + chemistry=Chemistry(args.chemistry), ) # begin readfish function From 283aa5f9e9fb5b21cb90ffbd82e9468ff63acd43 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 24 Jan 2024 21:16:25 +0000 Subject: [PATCH 04/33] Add duplex override decision and first read override decision --- src/readfish/plugins/utils.py | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index df964722..70a02bde 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -420,6 +420,10 @@ class Decision(Enum): above_max_chunks = "above_max_chunks" #: Fewer signal chunks for this read collected than required below_min_chunks = "below_min_chunks" + #: Potential second half of a duplex read + duplex_override = "duplex_override" + #: Read sequenced as translocated portion was of unknown length at start of readfish + first_read_override = "first_read_override" @unique @@ -773,6 +777,7 @@ class PreviouslySentActionTracker: """ last_actions: Dict[int, Action] = attrs.Factory(dict) + last_decision: Dict[int, Decision] = attrs.Factory(dict) def add_action(self, channel: int, action: Action) -> None: """ @@ -792,3 +797,80 @@ def get_action(self, channel: int) -> Optional[Action]: :return: The last action sent for the channel, or None if no action has been sent. """ return self.last_actions.get(channel, None) + + +@attrs.define +class DuplexTracker: + """ + Wrapper class to keep track the alignment location of the latest read seen on a channel, + and previous decision made, tracking whether we made a duplex override on the last read + Specifically, we store a list of tuples of any target contig names and strands that were aligned to, + keyed to channel number and the previous decision for a read made on that channel. + The decision should only be updated when a read has been finalised and should not be seen again, + i.e a Stop receiving or Unblock has been sent to MinKNOW + No maps are specified as (*, *) + """ + + previous_alignments: Dict[int, list[tuple[str, Strand]]] = attrs.Factory(dict) + previous_decision: Dict[int, Decision] = attrs.Factory(dict) + + def get_previous_decision(self, channel: int) -> Decision: + """ + Get the previous decision seen on this channel. + + :param channel: The channel number. + :return: Previously seen decision + """ + self.previous_decision.get(channel, None) + + def set_previous_decision(self, channel: int, decision: Decision) -> None: + """ + Set the previous decision for a given channel number. + + :param channel: The channel number. + :param decision: The decision taken. Should be the final decision, + i.e we won't see the read again. + """ + self.previous_decision[channel] = decision + + def get_previous_alignment(self, channel: int) -> list[tuple[str, Strand]]: + """ + Retrieves last alignment, including no maps seen on the given channel. + + :param channel: The channel number to lookup the previous action for + :param read_id: Read of ID of the current alignment + :return: Returns a tuple of (contig_name, strand), for the last alignment seen on this channel + """ + return self.previous_alignments.get(channel, None) + + def add_alignments( + self, channel: int, alignments: list[tuple[str, Strand]] + ) -> None: + """ + Add an alignment that has been seen for a channel. + + :param channel: The channel number to set the alignment for. + :param target_name: The name of the target contig aligned to + :param strand: The strand we have aligned to. + """ + self.previous_alignments[channel] = alignments + + def possible_duplex(self, channel: int, target_name: str, strand: Strand) -> bool: + """ + Compare the current alignment target_name and strand for a given channel + with the previous alignment target_name and strand. + + If the strand is opposite and the target is the same, return True, else False. + :param channel: Channel number to fetch alignment for + :param target_name: The name of the target contig for the current alignment + :param strand: The strand of the current alignment + :return: True if the strand is opposite and target contig the same + """ + return any( + prev_alignment + == ( + target_name, + Strand.forward if strand == Strand.reverse else Strand.reverse, + ) + for prev_alignment in self.get_previous_alignment(channel) + ) From 2a67a4c39f7dc5c5748bd7bdb8b0f62039da03f7 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 24 Jan 2024 21:16:49 +0000 Subject: [PATCH 05/33] Add Chemistry flag to cli args --- src/readfish/_cli_args.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/readfish/_cli_args.py b/src/readfish/_cli_args.py index 521f3c46..008f39da 100644 --- a/src/readfish/_cli_args.py +++ b/src/readfish/_cli_args.py @@ -5,8 +5,18 @@ ``BASE_ARGS`` are the minimal required arguments for _all_ entry points as they used for initialising loggers. ``DEVICE_BASE_ARGS`` are the set of arguments that are used for connecting to a sequencer (device) and some other related settings for selective sequencing scripts. """ + +from enum import Enum, unique from readfish._utils import nice_join + +@unique +class Chemistry(Enum): + DUPLEX = "duplex" + SIMPLEX = "simplex" + DUPLEX_SIMPLE = "duplex_simple" + + DEFAULT_SERVER_HOST = "127.0.0.1" DEFAULT_SERVER_PORT = None DEFAULT_LOG_FORMAT = "%(asctime)s %(name)s %(message)s" @@ -134,11 +144,14 @@ ), ), ( - "--duplex", + "--chemistry", dict( - help="**EXPERIMENTAL** Enable duplex targets mode. Accepts reads that align to the opposite strand and same contig as the previous read on channel.", + help="**EXPERIMENTAL** Choose between duplex and simplex chemistry mode. duplex_simple accept a read if the previous channels read was stop receiving," + "duplex checks that the previous reads alignment was on the same contig and opposite strand. default: SIMPLEX", required=False, - action="store_true", + type=str, + default=Chemistry.SIMPLEX, + choices=[chemistry.name for chemistry in Chemistry], ), ), ) + BASE_ARGS From 36a428e5d5b3b3167462982f7e7a427d7a005799 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 24 Jan 2024 21:19:44 +0000 Subject: [PATCH 06/33] Include readfish version in printargs ooutput --- src/readfish/_loggers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/readfish/_loggers.py b/src/readfish/_loggers.py index c77a03f1..6ac3ad2d 100644 --- a/src/readfish/_loggers.py +++ b/src/readfish/_loggers.py @@ -4,6 +4,7 @@ from logging.handlers import QueueHandler, QueueListener import queue from typing import Callable +from readfish.__about__ import __version__ def setup_logger( @@ -109,3 +110,4 @@ def print_args( for attr in dir(args): if attr[0] != "_" and attr not in exclude and attr.lower() == attr: printer(f"{attr}={getattr(args, attr)!r}") + printer(f"Version={__version__}") From cec09dc51abfd7ee371ae7cf5b890c3e6ed6f4d2 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 24 Jan 2024 21:35:51 +0000 Subject: [PATCH 07/33] Update expected error messages for wrong reference types --- .../mappy_validation_test/fail/001_wrong_reference_type.txt | 2 +- .../fail/003_wrong_reference_type_mmi_gz.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt b/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt index 0193d70c..6d69825d 100644 --- a/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt +++ b/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt @@ -1 +1 @@ -Provided index file appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] +Provided index file (.txt) appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] \ No newline at end of file diff --git a/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt b/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt index 0193d70c..69930bd8 100644 --- a/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt +++ b/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt @@ -1 +1 @@ -Provided index file appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] +Provided index file (.mmi.gz) appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] \ No newline at end of file From c36c50e30094ee3e756d2c83c7145f371ba6390e Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 25 Jan 2024 09:45:08 +0000 Subject: [PATCH 08/33] Doctests on Duplex tracker class bumps coverage >72 --- src/readfish/plugins/utils.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index 70a02bde..be1c9e82 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -820,8 +820,14 @@ def get_previous_decision(self, channel: int) -> Decision: :param channel: The channel number. :return: Previously seen decision + >>> dt = DuplexTracker() + >>> dt.get_previous_decision(1) is None + True + >>> dt.set_previous_decision(1, Decision.duplex_override) + >>> dt.get_previous_decision(1) + """ - self.previous_decision.get(channel, None) + return self.previous_decision.get(channel, None) def set_previous_decision(self, channel: int, decision: Decision) -> None: """ @@ -830,6 +836,10 @@ def set_previous_decision(self, channel: int, decision: Decision) -> None: :param channel: The channel number. :param decision: The decision taken. Should be the final decision, i.e we won't see the read again. + >>> dt = DuplexTracker() + >>> dt.set_previous_decision(1, Decision.no_map) + >>> dt.previous_decision[1] + """ self.previous_decision[channel] = decision @@ -840,6 +850,13 @@ def get_previous_alignment(self, channel: int) -> list[tuple[str, Strand]]: :param channel: The channel number to lookup the previous action for :param read_id: Read of ID of the current alignment :return: Returns a tuple of (contig_name, strand), for the last alignment seen on this channel + + >>> dt = DuplexTracker() + >>> dt.get_previous_alignment(1) is None + True + >>> dt.add_alignments(1, [("contig1", Strand.forward), ("contig2", Strand.reverse)]) + >>> dt.get_previous_alignment(1) + [('contig1', ), ('contig2', )] """ return self.previous_alignments.get(channel, None) @@ -852,6 +869,11 @@ def add_alignments( :param channel: The channel number to set the alignment for. :param target_name: The name of the target contig aligned to :param strand: The strand we have aligned to. + + >>> dt = DuplexTracker() + >>> dt.add_alignments(1, [("contig3", Strand.forward), ("contig4", Strand.reverse)]) + >>> dt.previous_alignments[1] + [('contig3', ), ('contig4', )] """ self.previous_alignments[channel] = alignments @@ -865,6 +887,13 @@ def possible_duplex(self, channel: int, target_name: str, strand: Strand) -> boo :param target_name: The name of the target contig for the current alignment :param strand: The strand of the current alignment :return: True if the strand is opposite and target contig the same + + >>> dt = DuplexTracker() + >>> dt.add_alignments(1, [("contig5", Strand.forward)]) + >>> dt.possible_duplex(1, "contig5", Strand.reverse) + True + >>> dt.possible_duplex(1, "contig6", Strand.reverse) + False """ return any( prev_alignment From aa6aa6919e5cc20db72b985ea4f017c49c3503f3 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:06:45 +0000 Subject: [PATCH 09/33] Include chemistry on analysis class --- src/readfish/entry_points/targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index e9dd2e55..8c1684b9 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -167,7 +167,7 @@ def __init__( self.dry_run = dry_run self.live_toml = Path(f"{toml}_live").resolve() self.run_information = self.client.connection.protocol.get_run_info() - + self.chemistry = chemistry # Generate a run specific read log read_log_name = ( f"{self.run_information.run_id}_readfish.tsv" if debug_log else None From 4c41e0f0f4f93b979fe0cf690844d0d93a714e5d Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:06:53 +0000 Subject: [PATCH 10/33] Whoops --- src/readfish/_cli_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/_cli_args.py b/src/readfish/_cli_args.py index 008f39da..83fa0a36 100644 --- a/src/readfish/_cli_args.py +++ b/src/readfish/_cli_args.py @@ -151,7 +151,7 @@ class Chemistry(Enum): required=False, type=str, default=Chemistry.SIMPLEX, - choices=[chemistry.name for chemistry in Chemistry], + choices=[chemistry.value for chemistry in Chemistry], ), ), ) + BASE_ARGS From b4253d96ff1d42643608aaea05b68b4c23a27c52 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:08:07 +0000 Subject: [PATCH 11/33] bump version --- src/readfish/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/__about__.py b/src/readfish/__about__.py index 4e896e6f..9ba8864e 100644 --- a/src/readfish/__about__.py +++ b/src/readfish/__about__.py @@ -1,4 +1,4 @@ """__about__.py Version of the read until software """ -__version__ = "2023.1.1" +__version__ = "2024.2.0a" From a8b8a4cbafc87cde9ebef1a9446ae86ef62f63c9 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:12:10 +0000 Subject: [PATCH 12/33] I'mn trash ( include channel kwarg in duplex tracker) --- src/readfish/entry_points/targets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 8c1684b9..ffceb9a3 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -360,7 +360,7 @@ def check_override_action( # Add decided Action self.previous_action_tracker.add_action(result.channel, action) # Add final decision - used to check if it is a duplex override - self.duplex_tracker.set_previous_decision(result.decision) + self.duplex_tracker.set_previous_decision(result.channel, result.decision) elif action is Action.unblock: if self.dry_run: # Log an 'unblock' action to previous action, but send a 'stop receiving' to prevent further read processing. @@ -372,7 +372,7 @@ def check_override_action( ) # Add decided Action self.previous_action_tracker.add_action(result.channel, action) - self.duplex_tracker.set_previous_decision(result.decision) + self.duplex_tracker.set_previous_decision(result.channel, result.decision) return ( previous_action, From 0c2cebadba04af0be90fe1c9eb2aa45a482e9eaf Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:15:07 +0000 Subject: [PATCH 13/33] Comment out smart duplex alignment tracking --- src/readfish/entry_points/targets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index ffceb9a3..ed2e9a1c 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -481,9 +481,9 @@ def run(self): ), overridden_action_name=overridden_action_name, ) - self.previous_alignment_tracker.add_alignment( - result.channel, - ) + # self.duplex_tracker.add_alignments( + # result.channel, + # ) ####################################################################### # Compile actions to be sent From 0a353e62732f77686f7c6adbcd8324e4ed3ade28 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:18:36 +0000 Subject: [PATCH 14/33] result.channel on getting previous decision WHO wrote this literally crying --- src/readfish/entry_points/targets.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index ed2e9a1c..47c075b7 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -327,7 +327,8 @@ def check_override_action( and previous_action == Action.stop_receiving # And we aren't already sequencing it and action != Action.stop_receiving - and self.duplex_tracker.get_previous_decision() != Decision.duplex_override + and self.duplex_tracker.get_previous_decision(result.channel) + != Decision.duplex_override ): self.logger.debug( f"Overriding read {result.read_id} as it is possibly second half of a duplex" @@ -340,7 +341,8 @@ def check_override_action( self.chemistry == Chemistry.DUPLEX_SIMPLE and previous_action == Action.stop_receiving and action != Action.stop_receiving - and self.duplex_tracker.get_previous_decision() != Decision.duplex_override + and self.duplex_tracker.get_previous_decision(result.channel) + != Decision.duplex_override ): action = Action.stop_receiving action_overridden = True @@ -481,9 +483,10 @@ def run(self): ), overridden_action_name=overridden_action_name, ) - # self.duplex_tracker.add_alignments( - # result.channel, - # ) + # if self.chemistry == Chemistry.DUPLEX: + # self.duplex_tracker.add_alignments( + # result.channel, [()] + # ) ####################################################################### # Compile actions to be sent From d246600ba0a54caf4279178218d7365a91f26202 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:22:21 +0000 Subject: [PATCH 15/33] Dedent version printing out of for loop --- src/readfish/_loggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/_loggers.py b/src/readfish/_loggers.py index 6ac3ad2d..30897882 100644 --- a/src/readfish/_loggers.py +++ b/src/readfish/_loggers.py @@ -110,4 +110,4 @@ def print_args( for attr in dir(args): if attr[0] != "_" and attr not in exclude and attr.lower() == attr: printer(f"{attr}={getattr(args, attr)!r}") - printer(f"Version={__version__}") + printer(f"Version={__version__}") From cb29449fb5bc20677d2503ecade634bd846629c4 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:26:10 +0000 Subject: [PATCH 16/33] Add silly log message to be removed --- src/readfish/entry_points/targets.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 47c075b7..c1cc556f 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -343,7 +343,11 @@ def check_override_action( and action != Action.stop_receiving and self.duplex_tracker.get_previous_decision(result.channel) != Decision.duplex_override - ): + ): # TODO REMOVE + self.logger.info( + f"Overriding to duplex - previous read action {previous_action}, current_action: {action}," + f" previous_decision: {self.duplex_tracker.get_previous_decision(result.channel)}" + ) action = Action.stop_receiving action_overridden = True result.decision = Decision.duplex_override From 319f9cf187178e19e7150062c4edd7e9ac0a1777 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 13:56:09 +0000 Subject: [PATCH 17/33] On;y duplex if prev decision is not first read override or duplex_override, And we were going to unblock this one --- src/readfish/entry_points/targets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index c1cc556f..61e75274 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -128,6 +128,7 @@ ), ), ) +DISALLOWED_DUPLEX_DECISIONS = {Decision.first_read_override, Decision.duplex_override} class Analysis: @@ -315,6 +316,7 @@ def check_override_action( previous_action = self.previous_action_tracker.get_action(result.channel) action_overridden = False # If --duplex flag override decisions made based on the strand and contig alignment of the previous read. + # Unfinished bruv if ( self.chemistry == Chemistry.DUPLEX and any( @@ -340,10 +342,10 @@ def check_override_action( elif ( self.chemistry == Chemistry.DUPLEX_SIMPLE and previous_action == Action.stop_receiving - and action != Action.stop_receiving + and action == Action.unblock and self.duplex_tracker.get_previous_decision(result.channel) - != Decision.duplex_override - ): # TODO REMOVE + not in DISALLOWED_DUPLEX_DECISIONS + ): # TODO R self.logger.info( f"Overriding to duplex - previous read action {previous_action}, current_action: {action}," f" previous_decision: {self.duplex_tracker.get_previous_decision(result.channel)}" From 0597f030fcd6310cfaeea2ba661783ffb84b8e74 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 31 Jan 2024 14:05:44 +0000 Subject: [PATCH 18/33] Duplex override log message changed to debug --- src/readfish/entry_points/targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 61e75274..351eefd2 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -346,7 +346,7 @@ def check_override_action( and self.duplex_tracker.get_previous_decision(result.channel) not in DISALLOWED_DUPLEX_DECISIONS ): # TODO R - self.logger.info( + self.logger.debug( f"Overriding to duplex - previous read action {previous_action}, current_action: {action}," f" previous_decision: {self.duplex_tracker.get_previous_decision(result.channel)}" ) From 041c393da5b774c33924ccd1c823bdf6eda676ed Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 1 Feb 2024 10:29:14 +0000 Subject: [PATCH 19/33] Add __invert__ to strand So it can be inverted with bitwise NOT to the opposite strand --- src/readfish/plugins/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index be1c9e82..2d2bddd8 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -30,6 +30,16 @@ class Strand(Enum): #: Reverse strand reverse = "-" + def __invert__(self): + """Flip the strand, using bitwise NOT (~) + + >>> ~Strand.forward + + >>> ~Strand.reverse + + """ + return Strand.forward if self is Strand.reverse else Strand.reverse + STRANDS = { 1: Strand.forward, From bdee12b0c10813b2ad855ee1ed778aa16eaff32c Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 1 Feb 2024 11:17:46 +0000 Subject: [PATCH 20/33] Fix docstrings Fix comparisons of chemistry enum Delete unused code --- src/readfish/entry_points/targets.py | 39 +++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 351eefd2..83059055 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -97,6 +97,7 @@ PreviouslySentActionTracker, Result, DuplexTracker, + Strand, ) @@ -128,6 +129,8 @@ ), ), ) +# Whe sequencing in duplex mode, overriding a decided `Action` on a currently sequenced molecule +# is not allowed if the previous molecules decision was one of these. DISALLOWED_DUPLEX_DECISIONS = {Decision.first_read_override, Decision.duplex_override} @@ -137,14 +140,16 @@ class Analysis: function that is run threaded in the run function at the base of this file. Arguments listed in the __init__ docs. - :param client: An instance of the ReadUntilClient object - :param conf: An instance of the Conf object - :param logger: The command level logger for this module - :param debug_log: Whether to output the Debug Log. log Name is generated. Defaults to False. - :param throttle: The number of seconds interval between requests to the ReadUntilClient, defaults to 0.1 - :param unblock_duration: Time, in seconds, to apply unblock voltage, defaults to 0.5 - :param dry_run: If True unblocks are replaced with `stop_receiving` commands, defaults to False - :param toml: The path to the toml file containing experiment conf. Used for reloading, defaults to "a.toml" + :param client: An instance of the ReadUntilClient object. + :param conf: An instance of the Conf object. + :param logger: The command level logger for this module. + :param debug_log: Whether to output the Debug Log. log Name is generated. + :param throttle: The time interval (seconds) between requests to the ReadUntilClient. + :param unblock_duration: Time, in seconds, to apply unblock voltage. + :param dry_run: If True unblocks are replaced with `stop_receiving` commands. + :param toml: The path to the toml file containing experiment conf. Used as the path for checking if the TOML needs reloading. + :param chemistry: Instance of Chemistry Enum, representing the chemistry of the run (Simplex/Duplex). Used for + decision making on strands that may be part of a duplex pair. """ def __init__( @@ -272,9 +277,11 @@ def check_override_action( Checks include: 1. If the read is in a control region, the action is always stop_receiving. 1. If the read is below the minimum chunks, use value in toml or default to proceed - 1. If the read is above the maximum chunks, use value in toml or default unblock + 1. If the read is above the maximum chunks, use value in toml or default unblock--throttle 1. First read seen for channel and readfish started during sequencing, override to stop_receiving 1. If action is unblock and we are dry-running, override to stop_receiving + 1. If we are running in duplex chemistry, check the previous reads final decision and Action, and potentially sequence + the current read, instead of unblocking it. :param control: Indicates read from a channel in a control region :param action: What action was decided for this read before any meddling @@ -318,10 +325,10 @@ def check_override_action( # If --duplex flag override decisions made based on the strand and contig alignment of the previous read. # Unfinished bruv if ( - self.chemistry == Chemistry.DUPLEX + self.chemistry is Chemistry.DUPLEX and any( self.previous_alignment_tracker.possible_duplex( - result.channel, result.read_id, al.ctg, al.strand + result.channel, result.read_id, al.ctg, Strand(al.strand) ) for al in result.alignment_data ) @@ -340,7 +347,7 @@ def check_override_action( action = Action.stop_receiving # Duplex elif ( - self.chemistry == Chemistry.DUPLEX_SIMPLE + self.chemistry is Chemistry.DUPLEX_SIMPLE and previous_action == Action.stop_receiving and action == Action.unblock and self.duplex_tracker.get_previous_decision(result.channel) @@ -368,7 +375,7 @@ def check_override_action( # Add decided Action self.previous_action_tracker.add_action(result.channel, action) # Add final decision - used to check if it is a duplex override - self.duplex_tracker.set_previous_decision(result.channel, result.decision) + self.duplex_tracker.set_decision(result.channel, result.decision) elif action is Action.unblock: if self.dry_run: # Log an 'unblock' action to previous action, but send a 'stop receiving' to prevent further read processing. @@ -380,7 +387,7 @@ def check_override_action( ) # Add decided Action self.previous_action_tracker.add_action(result.channel, action) - self.duplex_tracker.set_previous_decision(result.channel, result.decision) + self.duplex_tracker.set_decision(result.channel, result.decision) return ( previous_action, @@ -489,10 +496,6 @@ def run(self): ), overridden_action_name=overridden_action_name, ) - # if self.chemistry == Chemistry.DUPLEX: - # self.duplex_tracker.add_alignments( - # result.channel, [()] - # ) ####################################################################### # Compile actions to be sent From ce05dad5a89d7cd3f9ae80383c705a9abfc111f2 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 1 Feb 2024 11:18:27 +0000 Subject: [PATCH 21/33] Rename the duplextracker functions to be consistent with get/set use new bitwise NOT on strand for possible duplex --- src/readfish/_cli_args.py | 3 +++ src/readfish/plugins/utils.py | 34 ++++++++++++++++------------------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/readfish/_cli_args.py b/src/readfish/_cli_args.py index 83fa0a36..18cff940 100644 --- a/src/readfish/_cli_args.py +++ b/src/readfish/_cli_args.py @@ -12,8 +12,11 @@ @unique class Chemistry(Enum): + #: For the "smarter" version of duplex - does this read map to the previous reads opposite strand on the same contig. Won't work for no map based decisions DUPLEX = "duplex" + #: Normal simplex chemistry - no duplex override shenanigans SIMPLEX = "simplex" + #: Simple duplex - if we are going to unblock a read given the previous read on the same channel was stop receiving, sequence the current read instead. DUPLEX_SIMPLE = "duplex_simple" diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index 2d2bddd8..46be642a 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -787,7 +787,6 @@ class PreviouslySentActionTracker: """ last_actions: Dict[int, Action] = attrs.Factory(dict) - last_decision: Dict[int, Decision] = attrs.Factory(dict) def add_action(self, channel: int, action: Action) -> None: """ @@ -833,13 +832,13 @@ def get_previous_decision(self, channel: int) -> Decision: >>> dt = DuplexTracker() >>> dt.get_previous_decision(1) is None True - >>> dt.set_previous_decision(1, Decision.duplex_override) + >>> dt.set_decision(1, Decision.duplex_override) >>> dt.get_previous_decision(1) """ return self.previous_decision.get(channel, None) - def set_previous_decision(self, channel: int, decision: Decision) -> None: + def set_decision(self, channel: int, decision: Decision) -> None: """ Set the previous decision for a given channel number. @@ -847,30 +846,30 @@ def set_previous_decision(self, channel: int, decision: Decision) -> None: :param decision: The decision taken. Should be the final decision, i.e we won't see the read again. >>> dt = DuplexTracker() - >>> dt.set_previous_decision(1, Decision.no_map) + >>> dt.set_decision(1, Decision.no_map) >>> dt.previous_decision[1] """ self.previous_decision[channel] = decision - def get_previous_alignment(self, channel: int) -> list[tuple[str, Strand]]: + def get_previous_alignments(self, channel: int) -> list[tuple[str, Strand]]: """ - Retrieves last alignment, including no maps seen on the given channel. + Retrieves last alignments, including no maps seen on the given channel. :param channel: The channel number to lookup the previous action for :param read_id: Read of ID of the current alignment :return: Returns a tuple of (contig_name, strand), for the last alignment seen on this channel >>> dt = DuplexTracker() - >>> dt.get_previous_alignment(1) is None + >>> dt.get_previous_alignments(1) is None True - >>> dt.add_alignments(1, [("contig1", Strand.forward), ("contig2", Strand.reverse)]) - >>> dt.get_previous_alignment(1) + >>> dt.set_alignments(1, [("contig1", Strand.forward), ("contig2", Strand.reverse)]) + >>> dt.get_previous_alignments(1) [('contig1', ), ('contig2', )] """ return self.previous_alignments.get(channel, None) - def add_alignments( + def set_alignments( self, channel: int, alignments: list[tuple[str, Strand]] ) -> None: """ @@ -881,13 +880,15 @@ def add_alignments( :param strand: The strand we have aligned to. >>> dt = DuplexTracker() - >>> dt.add_alignments(1, [("contig3", Strand.forward), ("contig4", Strand.reverse)]) + >>> dt.set_alignments(1, [("contig3", Strand.forward), ("contig4", Strand.reverse)]) >>> dt.previous_alignments[1] [('contig3', ), ('contig4', )] """ self.previous_alignments[channel] = alignments - def possible_duplex(self, channel: int, target_name: str, strand: Strand) -> bool: + def possible_duplex( + self, channel: int, target_name: str, strand: Strand | str | int + ) -> bool: """ Compare the current alignment target_name and strand for a given channel with the previous alignment target_name and strand. @@ -899,17 +900,14 @@ def possible_duplex(self, channel: int, target_name: str, strand: Strand) -> boo :return: True if the strand is opposite and target contig the same >>> dt = DuplexTracker() - >>> dt.add_alignments(1, [("contig5", Strand.forward)]) + >>> dt.set_alignments(1, [("contig5", Strand.forward)]) >>> dt.possible_duplex(1, "contig5", Strand.reverse) True >>> dt.possible_duplex(1, "contig6", Strand.reverse) False """ + strand = Strand(strand) return any( - prev_alignment - == ( - target_name, - Strand.forward if strand == Strand.reverse else Strand.reverse, - ) + prev_alignment == (target_name, ~strand) for prev_alignment in self.get_previous_alignment(channel) ) From b705d0bae09f7ba34d7f84111dfbe7974039f214 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 1 Feb 2024 11:43:06 +0000 Subject: [PATCH 22/33] Add note for future --- src/readfish/plugins/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index 46be642a..ec017dbe 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -820,6 +820,9 @@ class DuplexTracker: No maps are specified as (*, *) """ + # Note - `readfish.src.plugins.utils.Alignment` could be used here, instead of tuple[str, Strand] + # We could then use the results of the ALignment directly, if we ever wanted to do something more complex + # for duplex previous_alignments: Dict[int, list[tuple[str, Strand]]] = attrs.Factory(dict) previous_decision: Dict[int, Decision] = attrs.Factory(dict) From 832e77506a1c7480cda4eee93c38d54657075cfe Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 1 Feb 2024 12:50:28 +0000 Subject: [PATCH 23/33] Split complex condition into 2 parts and move into more human friendly variable names --- src/readfish/entry_points/targets.py | 55 ++++++++++++++++------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 83059055..42e0a083 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -326,40 +326,47 @@ def check_override_action( # Unfinished bruv if ( self.chemistry is Chemistry.DUPLEX - and any( + # Easy checks first, so wdon't do more complex processing unless we have to + and action == Action.unblock + and previous_action is Action.stop_receiving + ): + # Check if we think this read is possibly duplex + possible_duplex = any( self.previous_alignment_tracker.possible_duplex( result.channel, result.read_id, al.ctg, Strand(al.strand) ) for al in result.alignment_data ) - # Previous action was to sequence - and previous_action == Action.stop_receiving - # And we aren't already sequencing it - and action != Action.stop_receiving - and self.duplex_tracker.get_previous_decision(result.channel) - != Decision.duplex_override - ): - self.logger.debug( - f"Overriding read {result.read_id} as it is possibly second half of a duplex" + # Check the previous decision for this channel was not already an override + previous_decision_allowed = ( + self.duplex_tracker.get_previous_decision(result.channel) + not in DISALLOWED_DUPLEX_DECISIONS ) - action_overridden = True - result.decision = Decision.duplex_on - action = Action.stop_receiving + if possible_duplex and previous_decision_allowed: + self.logger.debug( + f"Overriding read {result.read_id} as it is possibly second half of a duplex" + ) + action_overridden = True + result.decision = Decision.duplex_on + action = Action.stop_receiving # Duplex elif ( self.chemistry is Chemistry.DUPLEX_SIMPLE - and previous_action == Action.stop_receiving - and action == Action.unblock - and self.duplex_tracker.get_previous_decision(result.channel) - not in DISALLOWED_DUPLEX_DECISIONS - ): # TODO R - self.logger.debug( - f"Overriding to duplex - previous read action {previous_action}, current_action: {action}," - f" previous_decision: {self.duplex_tracker.get_previous_decision(result.channel)}" + and previous_action is Action.stop_receiving + and action is Action.unblock + ): + previous_decision_allowed = ( + self.duplex_tracker.get_previous_decision(result.channel) + not in DISALLOWED_DUPLEX_DECISIONS ) - action = Action.stop_receiving - action_overridden = True - result.decision = Decision.duplex_override + if previous_decision_allowed: + self.logger.debug( + f"Overriding to duplex - previous read action {previous_action}, current_action: {action}," + f" previous_decision: {self.duplex_tracker.get_previous_decision(result.channel)}" + ) + action = Action.stop_receiving + action_overridden = True + result.decision = Decision.duplex_override # Override to stop receiving if this is the first read ona channel and we started mid sequencing if previous_action is None and self.readfish_started_during_sequencing: From 0505d254bf505e152777a41aee3503371efb717a Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 1 Feb 2024 15:25:32 +0000 Subject: [PATCH 24/33] Fix typo'd variable Fix typod variable --- src/readfish/entry_points/targets.py | 2 +- src/readfish/plugins/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 42e0a083..9902d8f3 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -332,7 +332,7 @@ def check_override_action( ): # Check if we think this read is possibly duplex possible_duplex = any( - self.previous_alignment_tracker.possible_duplex( + self.duplex_tracker.possible_duplex( result.channel, result.read_id, al.ctg, Strand(al.strand) ) for al in result.alignment_data diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index ec017dbe..ee161253 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -912,5 +912,5 @@ def possible_duplex( strand = Strand(strand) return any( prev_alignment == (target_name, ~strand) - for prev_alignment in self.get_previous_alignment(channel) + for prev_alignment in self.get_previous_alignments(channel) ) From eb60807c2c99a6d9971eaf3de411a2f845f34a04 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Mon, 12 Feb 2024 15:02:23 +0000 Subject: [PATCH 25/33] Update non simple duplex tracking Move storing duplex tracker alignments/decisions into it's own conditional block --- src/readfish/entry_points/targets.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 9902d8f3..a753765e 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -66,6 +66,7 @@ """ + # Core imports from __future__ import annotations import argparse @@ -333,7 +334,7 @@ def check_override_action( # Check if we think this read is possibly duplex possible_duplex = any( self.duplex_tracker.possible_duplex( - result.channel, result.read_id, al.ctg, Strand(al.strand) + result.channel, result.read_id, al.ctg, al.strand ) for al in result.alignment_data ) @@ -345,9 +346,11 @@ def check_override_action( if possible_duplex and previous_decision_allowed: self.logger.debug( f"Overriding read {result.read_id} as it is possibly second half of a duplex" + f"- previous read action {previous_action}, current_action: {action}," + f" previous_decision: {self.duplex_tracker.get_previous_decision(result.channel)}" ) action_overridden = True - result.decision = Decision.duplex_on + result.decision = Decision.duplex_override action = Action.stop_receiving # Duplex elif ( @@ -379,10 +382,7 @@ def check_override_action( if action is Action.stop_receiving: stop_receiving_action_list.append((result.channel, result.read_number)) - # Add decided Action - self.previous_action_tracker.add_action(result.channel, action) - # Add final decision - used to check if it is a duplex override - self.duplex_tracker.set_decision(result.channel, result.decision) + elif action is Action.unblock: if self.dry_run: # Log an 'unblock' action to previous action, but send a 'stop receiving' to prevent further read processing. @@ -392,9 +392,20 @@ def check_override_action( unblock_batch_action_list.append( (result.channel, result.read_number, result.read_id) ) + + # If we have made a final decision for this read and we shouldn't see it again! + if action is Action.unblock or Action is Action.stop_receiving: # Add decided Action self.previous_action_tracker.add_action(result.channel, action) - self.duplex_tracker.set_decision(result.channel, result.decision) + # Add duplex based tracking if we are in duplex mode + if self.chemistry is Chemistry.DUPLEX_SIMPLE: + self.duplex_tracker.set_decision(result.channel, result.decision) + elif self.chemistry is Chemistry.DUPLEX: + self.duplex_tracker.set_decision(result.channel, result.decision) + self.duplex_tracker.set_alignments( + result.channel, + [(al.ctg, Strand(al.strand)) for al in result.alignment_data], + ) return ( previous_action, From 533b8462c77c588b59ebee1df3b3e62bf1a6fa34 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Mon, 12 Feb 2024 15:53:35 +0000 Subject: [PATCH 26/33] Fix typo --- src/readfish/entry_points/targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index a753765e..f195eb14 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -130,7 +130,7 @@ ), ), ) -# Whe sequencing in duplex mode, overriding a decided `Action` on a currently sequenced molecule +# When sequencing in duplex mode, overriding a decided `Action` on a currently sequenced molecule # is not allowed if the previous molecules decision was one of these. DISALLOWED_DUPLEX_DECISIONS = {Decision.first_read_override, Decision.duplex_override} From 0e823bdd8b920099c687e35e9d91eed975c89f23 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Mon, 12 Feb 2024 15:54:45 +0000 Subject: [PATCH 27/33] Remove comment about duplex being unfinished --- src/readfish/plugins/utils.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/readfish/plugins/utils.py b/src/readfish/plugins/utils.py index ee161253..b07684c7 100644 --- a/src/readfish/plugins/utils.py +++ b/src/readfish/plugins/utils.py @@ -170,11 +170,16 @@ def count_dict_elements(d: dict[Any]) -> int: """ return sum( ( - count_dict_elements(v) if isinstance(v, dict) - # If v is a list, tuple, sequence, dict etc., return the length of the container filtering out any empty sun elements, - else len(list(filterfalse(partial(is_empty), v))) - if isinstance(v, Container) - else 1 + ( + count_dict_elements(v) + if isinstance(v, dict) + # If v is a list, tuple, sequence, dict etc., return the length of the container filtering out any empty sun elements, + else ( + len(list(filterfalse(partial(is_empty), v))) + if isinstance(v, Container) + else 1 + ) + ) for v in d.values() ) ) From acd6c65974219f4a0419a85999edbaeef92f355b Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Mon, 12 Feb 2024 16:07:02 +0000 Subject: [PATCH 28/33] Update expected messages --- .../mappy_validation_test/fail/001_wrong_reference_type.txt | 2 +- .../fail/003_wrong_reference_type_mmi_gz.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt b/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt index 6d69825d..52f62746 100644 --- a/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt +++ b/tests/static/mappy_validation_test/fail/001_wrong_reference_type.txt @@ -1 +1 @@ -Provided index file (.txt) appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] \ No newline at end of file +Provided index file appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] \ No newline at end of file diff --git a/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt b/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt index 69930bd8..52f62746 100644 --- a/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt +++ b/tests/static/mappy_validation_test/fail/003_wrong_reference_type_mmi_gz.txt @@ -1 +1 @@ -Provided index file (.mmi.gz) appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] \ No newline at end of file +Provided index file appears to be of an incorrect type - should be one of ['.fasta', '.fna', '.fsa', '.fa', '.fastq', '.fq', '.fasta.gz', '.fna.gz', '.fsa.gz', '.fa.gz', '.fastq.gz', '.fq.gz', '.mmi'] \ No newline at end of file From 25ebfb1dcfd392884cb0906a0529c7358835df15 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Mon, 12 Feb 2024 17:50:20 +0000 Subject: [PATCH 29/33] Ignore targets.py for coverage --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e62ee3ac..5ff8c11d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,7 +47,7 @@ jobs: python -m site python -m pip install -U pip setuptools wheel python -m pip install -e ".[tests]" - coverage run --omit "src/readfish/read_until/*.py" -pm pytest + coverage run --omit "src/readfish/read_until/*.py,src/readfish/entry_points/targets.py" -pm pytest - name: Upload coverage data uses: actions/upload-artifact@v3 with: From edcf0b11f93f66c0d948b84d864f4a2378cfa46e Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Thu, 21 Mar 2024 15:56:18 +0000 Subject: [PATCH 30/33] big yikes - fixing typo in comparison to do log final decision! --- src/readfish/entry_points/targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index f195eb14..7940d2f6 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -394,7 +394,7 @@ def check_override_action( ) # If we have made a final decision for this read and we shouldn't see it again! - if action is Action.unblock or Action is Action.stop_receiving: + if action is Action.unblock or action is Action.stop_receiving: # Add decided Action self.previous_action_tracker.add_action(result.channel, action) # Add duplex based tracking if we are in duplex mode From 9d0d72548045dbc887ba6e4260ee384dee5c5f16 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 11:24:40 +0100 Subject: [PATCH 31/33] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c0a0978d..cb8585cc 100644 --- a/README.md +++ b/README.md @@ -555,6 +555,7 @@ And for our Awesome Logo please checkout out [@tim_bassford](https://twitter.com # Changelog ## Unreleased changes 1. Change the default `unblock_duration` on the `Analysis` class to use `DEFAULT_UNBLOCK` value defined in `_cli_args.py`. Change type on the Argparser for `--unblock-duration` to float. (#313) +1. Big dog Duplex feature - adds ability to select duplex reads that cover a target region. See pull request for details [(#324)](https://github.com/LooseLab/readfish/pull/324) ## 2023.1.1 1. Fix Readme Logo link 🥳 (#296) 1. Fix bug where we had accidentally started requiring barcoded TOMLs to specify a region. Thanks to @jamesemery for catching this. (#299) From 946ce9e29da56206385b0b0077349fb6fa15a905 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 12:32:03 +0100 Subject: [PATCH 32/33] fix typo in `--chemistry` help string --- src/readfish/_cli_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readfish/_cli_args.py b/src/readfish/_cli_args.py index 18cff940..898d92b8 100644 --- a/src/readfish/_cli_args.py +++ b/src/readfish/_cli_args.py @@ -149,7 +149,7 @@ class Chemistry(Enum): ( "--chemistry", dict( - help="**EXPERIMENTAL** Choose between duplex and simplex chemistry mode. duplex_simple accept a read if the previous channels read was stop receiving," + help="**EXPERIMENTAL** Choose between duplex and simplex chemistry mode. duplex_simple accepts a read if the previous channels read was stop receiving," "duplex checks that the previous reads alignment was on the same contig and opposite strand. default: SIMPLEX", required=False, type=str, From ec6807d65e2a8c0dfb77b79fb172301407574e38 Mon Sep 17 00:00:00 2001 From: Adoni5 Date: Wed, 17 Apr 2024 12:37:45 +0100 Subject: [PATCH 33/33] Update `readfish targets` docs to include a description of the duplex chemistry settings --- src/readfish/entry_points/targets.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/readfish/entry_points/targets.py b/src/readfish/entry_points/targets.py index 7940d2f6..a8ba1ea8 100644 --- a/src/readfish/entry_points/targets.py +++ b/src/readfish/entry_points/targets.py @@ -11,6 +11,10 @@ These chunks of raw data are processed by the basecaller to produce FASTA, which is then aligned against the chosen reference genome. The result of the alignment is used, along with the targets provided in the :doc:`TOML ` file, to make a decision on each read. +In the new **experimental** Duplex mode, it is possible to override a decision for a read based on the action taken for a previous read. +This is done by passing `--chemistry` and setting either duplex, or duplex simple. +`duplex_simple` accepts a read if the previous channels read was stop receiving, `duplex` checks that the previous reads alignment was on the same contig and opposite strand. +The default chemistry is simplex. Running this should result in a very short (<1kb, ideally 400-600 bases) unblock peak at the start of a read length histogram and longer sequenced reads. @@ -22,6 +26,15 @@ --log-file rf.log \\ --debug-log chunks.tsv +Example experimental duplex command:: + + readfish targets --device X3 \\ + --experiment-name "test" \\ + --toml my_exp.toml \\ + --log-file rf.log \\ + --debug-log chunks.tsv + --chemistry duplex + In the debug_log chunks.tsv file, if this argument is passed, each line represents detailed information about a batch of read signal that has been processed in an iteration.