From c3566dd547352060abc6f8d67157febd84f78b10 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 11:23:02 -0400 Subject: [PATCH 01/13] feat(desktop): add local dictation pipeline Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- desktop/src-tauri/src/dictation.rs | 252 ++++++++++++++++++++++++++++ desktop/src-tauri/src/huddle/stt.rs | 104 ++++++++---- desktop/src-tauri/src/lib.rs | 1 + 3 files changed, 326 insertions(+), 31 deletions(-) create mode 100644 desktop/src-tauri/src/dictation.rs diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000000..dc1aed2279 --- /dev/null +++ b/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,252 @@ +//! Local dictation pipeline — uses the Parakeet STT engine for offline +//! speech-to-text in the message composer. +//! +//! Unlike the huddle STT pipeline (which posts kind:9 events to the relay), +//! dictation emits transcribed text back to the frontend via Tauri events +//! so the composer can display it in real-time. +//! +//! Key differences from huddle STT: +//! - No TTS barge-in / echo gating (no agent voice in composer context) +//! - No shared huddle PTT gate; the composer controls its own session +//! - Slightly longer silence threshold for more coherent sentences +//! - Text goes to the frontend, not to the relay + +use std::sync::{Arc, LazyLock, Mutex}; + +use tauri::{Emitter, State}; + +use crate::app_state::AppState; +use crate::huddle::{models, stt::SttPipeline}; + +/// Tauri event name emitted when a dictation transcript segment is ready. +const DICTATION_TRANSCRIPT_EVENT: &str = "dictation-transcript"; + +/// Tauri event name emitted when dictation state changes (started/stopped). +const DICTATION_STATE_EVENT: &str = "dictation-state"; + +/// State for the active dictation session. +/// +/// Stored app-wide behind a `Mutex`. Only one dictation session can be active +/// at a time (starting a new one stops the previous). +pub(crate) struct DictationState { + /// The running STT engine, if dictation is active. + engine: Option>, + /// Monotonically increasing session counter. Included in all emitted events + /// so the frontend can ignore stale transcripts from a previous session's + /// forwarder that arrive after a new session has started. + session_id: u64, +} + +impl DictationState { + pub fn new() -> Self { + Self { + engine: None, + session_id: 0, + } + } +} + +static DICTATION_STATE: LazyLock> = + LazyLock::new(|| Mutex::new(DictationState::new())); + +/// `start_dictation` — begin local STT dictation. +/// +/// Starts the Parakeet STT engine and spawns a task that emits +/// `dictation-transcript` events to the frontend as text is recognized. +/// Returns an error if models are not downloaded yet. +#[tauri::command] +pub async fn start_dictation(state: State<'_, AppState>) -> Result { + // Check if models are ready. + if !models::is_stt_ready() { + // Kick off download if not already in progress. + if let Some(mgr) = models::global_model_manager() { + mgr.start_stt_download(state.http_client.clone()); + } + return Err("STT model not ready — download in progress".to_string()); + } + + let model_dir = models::stt_model_dir().ok_or("STT model directory not found")?; + + // Stop any existing dictation session first. + stop_dictation_inner(None); + + let (engine, text_rx) = SttPipeline::new_dictation(model_dir)?; + let engine = Arc::new(engine); + + // Store the engine in state and increment the session counter. + let session_id = { + let mut ds = DICTATION_STATE.lock().unwrap_or_else(|e| e.into_inner()); + ds.engine = Some(Arc::clone(&engine)); + ds.session_id += 1; + ds.session_id + }; + + // Spawn a task that forwards transcribed text to the frontend. + let app_handle = state + .app_handle + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + + if let Some(handle) = app_handle { + let _ = handle.emit( + DICTATION_STATE_EVENT, + serde_json::json!({ "state": "started", "session": session_id }), + ); + spawn_dictation_forwarder(text_rx, handle, session_id); + } + + Ok(session_id) +} + +/// `stop_dictation` — stop the active dictation session. +/// +/// The final transcript (if any) is emitted asynchronously by the forwarder +/// task. The `dictation-state: stopped` event is emitted by the forwarder +/// after all pending transcripts have been forwarded, ensuring the frontend +/// receives the final text before the stopped signal. +/// +/// `session` scopes the stop to a specific session: the engine is only torn +/// down when the currently-stored `session_id` matches. This prevents a +/// delayed/fire-and-forget stop from an old session (e.g. one deferred behind +/// a final audio flush) from killing a *newer* session the user started in the +/// meantime. Pass `None` for an unconditional stop (used on cancel/unmount). +#[tauri::command] +pub fn stop_dictation(session: Option) -> Result<(), String> { + stop_dictation_inner(session); + // Note: `stopped` is emitted by the forwarder task after draining all + // pending transcripts — not here. This avoids a race where the frontend + // sees `stopped` before the final transcript arrives. + Ok(()) +} + +/// `push_dictation_audio` — feed raw PCM bytes into the dictation pipeline. +/// +/// Expects a raw binary body: an 8-byte little-endian `u64` session header +/// followed by f32 LE samples at 48 kHz mono. The header scopes the push to a +/// specific session — bytes are only fed to the engine when the header matches +/// the currently-stored `session_id`. This prevents late audio from a +/// just-stopped session (whose final `flushAudioBatch()` chunks are still +/// arriving) from being accepted by a *newer* session the user started in the +/// meantime and transcribed into the new draft. +/// +/// If no dictation session is active, or the session header doesn't match the +/// active session, the bytes are silently discarded. +#[tauri::command] +pub fn push_dictation_audio(request: tauri::ipc::Request<'_>) -> Result<(), String> { + /// Size of the leading little-endian `u64` session header. + const SESSION_HEADER_BYTES: usize = 8; + /// Maximum IPC audio batch size (audio payload only, excluding the header): 100 KB. + const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() < SESSION_HEADER_BYTES { + return Err(format!( + "audio batch too small: {} bytes (need at least {} for session header)", + bytes.len(), + SESSION_HEADER_BYTES + )); + } + let (header, audio) = bytes.split_at(SESSION_HEADER_BYTES); + if audio.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + audio.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + // `split_at` guarantees `header` is exactly `SESSION_HEADER_BYTES` long. + let session = u64::from_le_bytes( + header + .try_into() + .map_err(|_| "invalid session header".to_string())?, + ); + let ds = DICTATION_STATE.lock().unwrap_or_else(|e| e.into_inner()); + // Only feed audio tagged with the currently-active session. Late + // chunks from an old session are silently dropped. + if session == ds.session_id { + if let Some(ref engine) = ds.engine { + engine.push_audio(audio.to_vec())?; + } + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} + +/// `get_dictation_status` — check if local dictation is available and/or active. +#[tauri::command] +pub fn get_dictation_status() -> DictationStatus { + let model_ready = models::is_stt_ready(); + let is_active = DICTATION_STATE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .engine + .is_some(); + + DictationStatus { + available: model_ready, + active: is_active, + } +} + +/// Response for `get_dictation_status`. +#[derive(serde::Serialize, Clone)] +pub struct DictationStatus { + /// Whether the local STT model is downloaded and ready. + pub available: bool, + /// Whether a dictation session is currently active. + pub active: bool, +} + +// ── Internal helpers ────────────────────────────────────────────────────────── + +fn stop_dictation_inner(session: Option) { + let old_engine = { + let mut ds = DICTATION_STATE.lock().unwrap_or_else(|e| e.into_inner()); + // Session-scoped stop: only tear down when the requested session matches + // the one currently stored. A `None` session stops unconditionally. + match session { + Some(requested) if requested != ds.session_id => None, + _ => ds.engine.take(), + } + }; + if let Some(engine) = old_engine { + engine.shutdown(); + // Drop outside the lock — thread join may block briefly. + drop(engine); + } +} + +/// Spawn an async task that reads transcribed text and emits Tauri events. +/// +/// Each event includes the `session` ID so the frontend can ignore stale +/// transcripts from a previous session's forwarder. When the channel closes +/// (engine stopped), the forwarder emits `dictation-state: stopped`. +fn spawn_dictation_forwarder( + mut text_rx: tokio::sync::mpsc::Receiver, + app_handle: tauri::AppHandle, + session_id: u64, +) { + tauri::async_runtime::spawn(async move { + while let Some(text) = text_rx.recv().await { + if text.is_empty() { + continue; + } + let payload = serde_json::json!({ "text": text, "session": session_id }); + if app_handle + .emit(DICTATION_TRANSCRIPT_EVENT, payload) + .is_err() + { + break; // App window closed. + } + } + // All transcripts forwarded — signal the frontend that dictation is done. + let _ = app_handle.emit( + DICTATION_STATE_EVENT, + serde_json::json!({ "state": "stopped", "session": session_id }), + ); + }); +} diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 6f502ca72c..67b639332a 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -41,6 +41,23 @@ const AUDIO_QUEUE_DEPTH: usize = 50; /// Prevents OOM if VAD stays in speech mode (noisy environment). const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; +/// Dictation waits slightly longer for a natural pause before flushing. +const DICTATION_SILENCE_FLUSH_FRAMES: usize = 25; + +/// Emit a partial dictation result after two seconds of uninterrupted speech. +const DICTATION_PARTIAL_FLUSH_SAMPLES: usize = 16_000 * 2; + +struct SttPipelineConfig { + model_dir: PathBuf, + silence_flush_frames: usize, + max_speech_samples: usize, + partial_flush_samples: Option, + tts_active: Arc, + tts_cancel: Option>, + ptt_active: Option>, + flush_on_shutdown: bool, +} + /// Handle to the running STT pipeline. /// /// Not Clone — wrap in `Arc` to share across threads. @@ -88,27 +105,46 @@ impl SttPipeline { tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + Self::new_with_config(SttPipelineConfig { + model_dir, + silence_flush_frames: SILENCE_FLUSH_FRAMES, + max_speech_samples: MAX_SPEECH_SAMPLES, + partial_flush_samples: None, + tts_active, + tts_cancel, + ptt_active, + flush_on_shutdown: false, + }) + } + + /// Spawn an offline STT pipeline for message-composer dictation. + pub(crate) fn new_dictation( + model_dir: PathBuf, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + Self::new_with_config(SttPipelineConfig { + model_dir, + silence_flush_frames: DICTATION_SILENCE_FLUSH_FRAMES, + max_speech_samples: MAX_SPEECH_SAMPLES, + partial_flush_samples: Some(DICTATION_PARTIAL_FLUSH_SAMPLES), + tts_active: Arc::new(AtomicBool::new(false)), + tts_cancel: None, + ptt_active: None, + flush_on_shutdown: true, + }) + } + + fn new_with_config( + config: SttPipelineConfig, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); let shutdown = Arc::new(AtomicBool::new(false)); let shutdown_worker = Arc::clone(&shutdown); - let tts_cancel_worker = tts_cancel.as_ref().map(Arc::clone); - let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); let handle = thread::Builder::new() .name("stt-worker".into()) - .spawn(move || { - stt_worker( - model_dir, - audio_rx, - text_tx, - shutdown_worker, - tts_active, - tts_cancel_worker, - ptt_active_worker, - ) - }) + .spawn(move || stt_worker(config, audio_rx, text_tx, shutdown_worker)) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; let pipeline = Self { @@ -202,13 +238,10 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(50); const STT_NUM_THREADS: i32 = 1; fn stt_worker( - model_dir: PathBuf, + config: SttPipelineConfig, audio_rx: Receiver>, text_tx: tokio_mpsc::Sender, shutdown: Arc, - tts_active: Arc, - tts_cancel: Option>, - ptt_active: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -235,12 +268,12 @@ fn stt_worker( // in k2-fsa/sherpa-onnx.) use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; - let tokens_path = model_dir.join("tokens.txt"); - let model_path = model_dir.join("model.int8.onnx"); + let tokens_path = config.model_dir.join("tokens.txt"); + let model_path = config.model_dir.join("model.int8.onnx"); if !tokens_path.exists() || !model_path.exists() { eprintln!( "buzz-desktop: STT model not found at {} — STT disabled", - model_dir.display() + config.model_dir.display() ); drain_until_shutdown(audio_rx, &shutdown); return; @@ -281,7 +314,8 @@ fn stt_worker( // ── 5. Main loop ────────────────────────────────────────────────────────── let mut tts_was_active = false; - let mut ptt_was_active = ptt_active + let mut ptt_was_active = config + .ptt_active .as_ref() .is_some_and(|p| p.load(Ordering::Acquire)); loop { @@ -291,7 +325,7 @@ fn stt_worker( } // Track TTS transitions to set the cooldown timer. - let tts_now = tts_active.load(Ordering::Acquire); + let tts_now = config.tts_active.load(Ordering::Acquire); if tts_was_active && !tts_now { // TTS just stopped — record the timestamp for the cooldown window. tts_stopped_at = Some(std::time::Instant::now()); @@ -302,7 +336,7 @@ fn stt_worker( // The worklet stops sending frames when PTT is inactive, so the normal // silence-accumulation flush path never runs. We must flush here on the // active→inactive edge to avoid buffering speech across PTT presses. - if let Some(ref ptt) = ptt_active { + if let Some(ref ptt) = config.ptt_active { let ptt_now = ptt.load(Ordering::Acquire); if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { flush_to_stt(&speech_buf, &recognizer, &text_tx); @@ -345,18 +379,21 @@ fn stt_worker( &mut barge_in_frames, &recognizer, &text_tx, - &tts_active, - tts_cancel.as_deref(), + &config.tts_active, + config.tts_cancel.as_deref(), &mut tts_stopped_at, - ptt_active.as_ref(), + config.ptt_active.as_ref(), + config.silence_flush_frames, + config.max_speech_samples, + config.partial_flush_samples, ); } } } - // No final flush — leave_huddle/end_huddle emit lifecycle events before - // the STT worker exits, so a final flush would post a kind:9 message AFTER - // the user has "left." Losing the last partial utterance is acceptable. + if config.flush_on_shutdown && !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + } } /// Resample a mono 48 kHz chunk to 16 kHz using rubato. @@ -411,6 +448,9 @@ fn process_16k_samples( tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, ptt_active: Option<&Arc>, + silence_flush_frames: usize, + max_speech_samples: usize, + partial_flush_samples: Option, ) { leftover.extend_from_slice(samples); @@ -494,7 +534,9 @@ fn process_16k_samples( speech_buf.extend_from_slice(&frame); // OOM guard: flush and reset if the buffer exceeds 30 s of audio. - if speech_buf.len() >= MAX_SPEECH_SAMPLES { + if speech_buf.len() >= max_speech_samples + || partial_flush_samples.is_some_and(|limit| speech_buf.len() >= limit) + { flush_to_stt(speech_buf, recognizer, text_tx); speech_buf.clear(); *silence_frames = 0; @@ -509,7 +551,7 @@ fn process_16k_samples( // key-hold as one utterance. The PTT release edge in the main // loop handles the flush. In VAD mode, flush after the silence // threshold so each natural pause becomes a separate message. - if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { + if ptt_active.is_none() && *silence_frames >= silence_flush_frames { // End of utterance — transcribe. flush_to_stt(speech_buf, recognizer, text_tx); speech_buf.clear(); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 6814008f0d..80b23b7b62 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -4,6 +4,7 @@ mod archive; mod builderlab; mod commands; mod deep_link; +mod dictation; mod egress_guard; mod event_sync; mod events; From 0367f751dd00e404f3831293ed798411656ad2b0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 11:23:23 -0400 Subject: [PATCH 02/13] feat(desktop): add voice dictation to composer Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- desktop/src/app/AppShell.tsx | 2 + .../dictation/hooks/useComposerDictation.ts | 191 ++++++ .../features/dictation/hooks/useDictation.ts | 90 +++ .../dictation/hooks/useLocalDictation.ts | 619 ++++++++++++++++++ .../hooks/useMessageComposerDictation.tsx | 49 ++ .../hooks/useVoiceDictationShortcut.ts | 53 ++ desktop/src/features/dictation/index.ts | 8 + .../features/dictation/lib/activeComposer.ts | 23 + .../dictation/lib/voiceInput.test.mjs | 150 +++++ .../src/features/dictation/lib/voiceInput.ts | 114 ++++ .../features/dictation/ui/DictationButton.tsx | 73 +++ .../features/messages/ui/MessageComposer.tsx | 98 +-- .../messages/ui/MessageComposerOverlays.tsx | 72 ++ .../messages/ui/useComposerScrollToBottom.ts | 13 + .../settings/ui/KeyboardShortcutsCard.tsx | 39 +- desktop/src/shared/lib/keyboard-shortcuts.ts | 8 + desktop/tests/e2e/messaging.spec.ts | 67 +- preview-features.json | 6 + 18 files changed, 1609 insertions(+), 66 deletions(-) create mode 100644 desktop/src/features/dictation/hooks/useComposerDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useLocalDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx create mode 100644 desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts create mode 100644 desktop/src/features/dictation/index.ts create mode 100644 desktop/src/features/dictation/lib/activeComposer.ts create mode 100644 desktop/src/features/dictation/lib/voiceInput.test.mjs create mode 100644 desktop/src/features/dictation/lib/voiceInput.ts create mode 100644 desktop/src/features/dictation/ui/DictationButton.tsx create mode 100644 desktop/src/features/messages/ui/MessageComposerOverlays.tsx create mode 100644 desktop/src/features/messages/ui/useComposerScrollToBottom.ts diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 4eb0a42bbe..fac09191bd 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -86,6 +86,7 @@ import { useIdentityQuery } from "@/shared/api/hooks"; import { useRelayAutoHeal } from "@/shared/api/useRelayAutoHeal"; import { useDeferredStartup } from "@/shared/hooks/useDeferredStartup"; import { useWebviewScrollBoundaryLock } from "@/shared/hooks/useWebviewScrollBoundaryLock"; +import { useVoiceDictationShortcut } from "@/features/dictation"; import { joinChannel } from "@/shared/api/tauri"; import type { ChannelVisibility, SearchHit } from "@/shared/api/types"; import { ChannelNavigationProvider } from "@/shared/context/ChannelNavigationContext"; @@ -153,6 +154,7 @@ export function AppShell() { }); // Settings lives in history so back returns to the previous app entry. const settingsOpen = location.pathname === "/settings"; + useVoiceDictationShortcut(settingsOpen); const locationSearchSection = (location.search as { section?: unknown }) .section; const settingsSection: SettingsSection = isSettingsSection( diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts new file mode 100644 index 0000000000..d11b2b3042 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -0,0 +1,191 @@ +import type * as React from "react"; +import { useEffect, useId, useRef } from "react"; +import { + clearActiveDictationComposer, + isActiveDictationComposer, + setActiveDictationComposer, +} from "../lib/activeComposer"; +import { useDictation } from "./useDictation"; + +interface UseComposerDictationOptions { + /** Whether the voice-dictation preview feature is enabled. */ + enabled?: boolean; + /** Ref to a function that syncs contentRef from the Tiptap editor and returns it. */ + syncContentRef: React.MutableRefObject<() => string>; + /** Whether the composer is currently disabled (read-only, etc.). */ + disabled?: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + /** Updates contentRef + isContentEmpty state. */ + setComposerContent: (text: string) => void; + /** Ref to a function that updates the Tiptap editor document. */ + setEditorContentRef: React.MutableRefObject<(text: string) => void>; + submitMessageRef: React.MutableRefObject<() => void>; + /** When this key changes (channel/thread switch), active dictation is stopped. */ + draftKey?: string | null; + /** Ref to the composer's container element for focus tracking. */ + composerRef?: React.RefObject; +} + +/** + * Thin wrapper around `useDictation` pre-wired for the MessageComposer's + * state management (syncContentRef, setComposerContent, editor, submitMessageRef). + * + * Uses the local Parakeet STT engine — fully offline, no relay or API key needed. + */ +export function useComposerDictation({ + enabled = true, + syncContentRef, + disabled = false, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + submitMessageRef, + draftKey, + composerRef, +}: UseComposerDictationOptions) { + const instanceId = useId(); + const isSendBlockedRef = useRef(false); + isSendBlockedRef.current = + !enabled || + disabledRef.current || + isSendingRef.current || + isUploadingRef.current; + + const dictation = useDictation({ + disabled: !enabled, + getText: () => syncContentRef.current(), + setText: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + }, + onSend: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + // Submit synchronously — the content ref is already set above, so + // syncComposerContentFromEditor() will serialize the editor which now + // holds the dictated text. + submitMessageRef.current(); + }, + isSendBlockedRef, + }); + + // Track which composer is active (most recently focused) so that the global + // ⌘D shortcut only dispatches to one instance when multiple are mounted. + useEffect(() => { + const el = composerRef?.current; + if (!el) { + // If no ref provided, this composer is always considered active (single-composer case). + setActiveDictationComposer(instanceId); + return () => clearActiveDictationComposer(instanceId); + } + + function handleFocusIn() { + setActiveDictationComposer(instanceId); + } + + el.addEventListener("focusin", handleFocusIn); + return () => { + el.removeEventListener("focusin", handleFocusIn); + clearActiveDictationComposer(instanceId); + }; + }, [instanceId, composerRef]); + + // Cancel dictation when the channel/thread changes so that transcript events + // from a stale local STT session don't leak into the wrong draft. + // Only cancel if this instance owns a live/pending session — avoids killing + // another composer's session since the native engine is a singleton. + // + // Includes `isTranscribing`, not just `isRecording`/`isStarting`: + // `stopRecording()` clears `isRecording` immediately but deliberately keeps + // the transcript listener alive (isTranscribing=true) until the native + // `stopped` event, so the final local STT flush can still be appended. If the + // draftKey changes during that grace window, we must still cancel — otherwise + // the late transcript is appended through this same composer instance into the + // newly restored draft. `cancelRecording()` unlistens the transcript handler + // and performs a session-scoped native stop, so this is safe. + const isOwningSessionRef = useRef(false); + isOwningSessionRef.current = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + // biome-ignore lint/correctness/useExhaustiveDependencies: draftKey is the sole trigger + useEffect(() => { + if (isOwningSessionRef.current) { + dictation.cancelRecording(); + } + }, [draftKey]); + + // Auto-cancel dictation when the composer becomes disabled mid-session + // (e.g. channel becomes read-only, parent send state disables thread composer). + // Without this, the STT session keeps running with no way to stop it. + // + // Covers the full ownership window — `isStarting` and `isTranscribing`, not + // just `isRecording`. If the composer is disabled while `startRecording()` is + // still resolving (mic permission / AudioWorklet setup), the pending start + // would otherwise finish and begin microphone capture in a disabled composer. + // `cancelRecording()` aborts the in-flight start as well as a live recording, + // and re-running when any of these flags flip catches a start that completes + // after the disable. + useEffect(() => { + const owningSession = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + if ((!enabled || disabled) && owningSession) { + dictation.cancelRecording(); + } + }, [ + enabled, + disabled, + dictation.isRecording, + dictation.isStarting, + dictation.isTranscribing, + dictation.cancelRecording, + ]); + + // ⌘D push-to-talk — hold to record, release to stop. + // Dispatched from AppShell's keydown/keyup handlers. + // Only the active (most recently focused) composer responds, and only while + // focus remains inside it and it is not disabled/send-blocked. + // biome-ignore lint/correctness/useExhaustiveDependencies: disabledRef/isSendBlockedRef are stable refs read at call time + useEffect(() => { + function handleKeyDown() { + if (!enabled) return; + // Only respond if this is the active composer instance. + if (!isActiveDictationComposer(instanceId)) return; + // Only respond if focus is still inside this composer. The active-composer + // registration persists after focusout (until unmount), so without this + // check, focusing a composer and then moving to another mounted input or + // dialog (quick search, create-channel, etc.) would still let ⌘D start + // microphone capture and append transcript into the background draft. + const el = composerRef?.current; + if (el && !el.contains(document.activeElement)) return; + // Don't start dictation in disabled/blocked composers. + if (disabledRef.current || isSendBlockedRef.current) return; + if (!dictation.isRecording && !dictation.isStarting) { + dictation.startRecording(); + } + } + function handleKeyUp() { + if (dictation.isRecording || dictation.isStarting) { + dictation.stopRecording(); + } + } + window.addEventListener("buzz:dictation-key-down", handleKeyDown); + window.addEventListener("buzz:dictation-key-up", handleKeyUp); + return () => { + window.removeEventListener("buzz:dictation-key-down", handleKeyDown); + window.removeEventListener("buzz:dictation-key-up", handleKeyUp); + }; + }, [ + instanceId, + composerRef, + enabled, + dictation.isRecording, + dictation.isStarting, + dictation.startRecording, + dictation.stopRecording, + ]); + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts new file mode 100644 index 0000000000..c11fc4b3be --- /dev/null +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -0,0 +1,90 @@ +import type * as React from "react"; +import { useCallback, useMemo, useRef } from "react"; +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "../lib/voiceInput"; +import { useLocalDictation } from "./useLocalDictation"; + +interface UseDictationOptions { + /** Disable native availability checks and recording entry points. */ + disabled?: boolean; + /** Returns the current composer text (must be fresh — synced from editor). */ + getText: () => string; + /** Set composer text */ + setText: (value: string) => void; + /** Send the message */ + onSend: (text: string) => void; + /** Ref that is `true` when sending is blocked (uploading, preparing mention, etc.) */ + isSendBlockedRef?: React.MutableRefObject; +} + +export function useDictation({ + disabled = false, + getText, + setText, + onSend, + isSendBlockedRef, +}: UseDictationOptions) { + const autoSubmitPhrases = useMemo( + () => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + [], + ); + const stopRecordingRef = useRef<() => void>(() => {}); + const lastTranscriptRef = useRef(""); + + const handleTranscript = useCallback( + (transcript: string) => { + const previous = lastTranscriptRef.current; + const latest = getText(); + const merged = replaceTrailingTranscribedText( + latest, + previous, + transcript, + ); + const match = getAutoSubmitMatch(transcript, autoSubmitPhrases); + + if (!match) { + setText(merged); + // Reset to empty — each streaming partial is an independent segment + // (the native engine flushes and clears its buffer). The next transcript + // should be appended, not replace this one. + lastTranscriptRef.current = ""; + return; + } + + const textWithoutPhrase = replaceTrailingTranscribedText( + latest, + previous, + match.textWithoutPhrase, + ); + if (!textWithoutPhrase.trim()) return; + + stopRecordingRef.current(); + + if (isSendBlockedRef?.current) { + setText(textWithoutPhrase); + return; + } + + setText(textWithoutPhrase.trim()); + onSend(textWithoutPhrase.trim()); + lastTranscriptRef.current = ""; + }, + [autoSubmitPhrases, getText, onSend, isSendBlockedRef, setText], + ); + + const dictation = useLocalDictation({ + disabled, + onRecordingStart: () => { + lastTranscriptRef.current = ""; + }, + onTranscriptText: handleTranscript, + }); + + stopRecordingRef.current = dictation.stopRecording; + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useLocalDictation.ts b/desktop/src/features/dictation/hooks/useLocalDictation.ts new file mode 100644 index 0000000000..2a4e3ba441 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useLocalDictation.ts @@ -0,0 +1,619 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; + +/** + * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. + * Same pattern as huddle's audioWorklet.ts. + */ +function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { + // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition + const internals = (window as any).__TAURI_INTERNALS__; + if (!internals?.invoke) { + return Promise.reject(new Error("Tauri internals not available")); + } + return internals.invoke(cmd, payload); +} + +interface UseLocalDictationOptions { + disabled?: boolean; + onRecordingStart?: () => void; + onTranscriptText: (text: string) => void; +} + +interface DictationStatus { + available: boolean; + active: boolean; +} + +const DICTATION_TRANSCRIPT_EVENT = "dictation-transcript"; +const DICTATION_STATE_EVENT = "dictation-state"; + +/** Interval (ms) to poll model availability after initial unavailability. */ +const MODEL_POLL_INTERVAL_MS = 5_000; + +/** + * Batching interval for audio IPC (ms). The worklet posts every render quantum + * (~2.67ms at 48kHz/128 samples). Sending each one individually overloads IPC. + * We accumulate ~100ms of audio before sending to reduce IPC overhead from + * ~375 calls/s to ~10 calls/s. + */ +const AUDIO_BATCH_MS = 100; + +/** + * Max samples per `push_dictation_audio` IPC call. The native command rejects + * any raw audio payload over 100 KB (`MAX_AUDIO_BATCH_BYTES` in `dictation.rs`); + * at 48 kHz f32 mono that is 25,600 samples (~0.53s). We chunk under that cap + * (24,000 samples / 96 KB, leaving headroom) so a stalled main thread that + * lets the batch grow past ~0.5s can't produce a single oversized buffer that + * native rejects and we silently drop. Chunks are sent in order. + */ +const MAX_IPC_SAMPLES = 24_000; + +/** + * Size (bytes) of the little-endian `u64` session header prepended to each + * `push_dictation_audio` payload. Native reads this header and only feeds audio + * whose session matches the currently-active one, so late chunks from a + * just-stopped session can't leak into a newer session's transcript. + */ +const SESSION_HEADER_BYTES = 8; + +/** + * Local STT dictation hook using the Parakeet model via Tauri native commands. + * + * Works fully offline — no relay or OpenAI API key needed. Uses the same + * sherpa-onnx Parakeet TDT-CTC 110M model as huddle transcription. + * + * Audio capture uses the Web Audio API (AudioWorklet) on the frontend side, + * then sends batched raw PCM bytes to the native STT engine via + * `push_dictation_audio`. + */ +export function useLocalDictation({ + disabled = false, + onRecordingStart, + onTranscriptText, +}: UseLocalDictationOptions) { + const [isRecording, setIsRecording] = useState(false); + const [isStarting, setIsStarting] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isAvailable, setIsAvailable] = useState(false); + + const streamRef = useRef(null); + const audioContextRef = useRef(null); + const workletRef = useRef(null); + const outputGainRef = useRef(null); + const batchTimerRef = useRef | null>(null); + const audioBatchRef = useRef([]); + // Tail of the serialized flush chain. Each flush appends its IPC work to + // this promise so flushes run strictly one-after-another; awaiting the tail + // guarantees every already-started flush's chunks are enqueued native-side. + const flushChainRef = useRef>(Promise.resolve()); + const unlistenTranscriptRef = useRef(null); + const unlistenStateRef = useRef(null); + const onRecordingStartRef = useRef(onRecordingStart); + const onTranscriptTextRef = useRef(onTranscriptText); + // Native session ID — set after `start_dictation` returns. Transcript and + // state events include this ID so we can definitively ignore stale events + // from a previous session's forwarder. + const nativeSessionRef = useRef(0); + // Abort flag — set when stop/cancel is called while startRecording is still + // awaiting async setup. The start resumes and bails before activating. + const startAbortedRef = useRef(false); + + onRecordingStartRef.current = onRecordingStart; + onTranscriptTextRef.current = onTranscriptText; + + const isEnabled = !disabled && isAvailable; + + // Check availability on mount and poll until available (model may be downloading). + useEffect(() => { + if (disabled) { + setIsAvailable(false); + return; + } + + let cancelled = false; + let pollTimer: ReturnType | null = null; + + function checkAvailability() { + invoke("get_dictation_status") + .then((status) => { + if (cancelled) return; + setIsAvailable(status.available); + // Stop polling once the model is ready. + if (status.available && pollTimer) { + clearInterval(pollTimer); + pollTimer = null; + } + }) + .catch(() => { + if (!cancelled) setIsAvailable(false); + }); + } + + checkAvailability(); + + // Poll periodically until available (handles background model download). + pollTimer = setInterval(() => { + if (cancelled) return; + checkAvailability(); + }, MODEL_POLL_INTERVAL_MS); + + return () => { + cancelled = true; + if (pollTimer) clearInterval(pollTimer); + }; + }, [disabled]); + + /** Flush accumulated audio batch to the native STT engine. Returns a promise + * that resolves once the IPC call completes (or immediately if nothing to flush). + * + * Flushes are serialized through `flushChainRef`: the batch is drained + * synchronously here (so each flush owns a distinct set of samples in FIFO + * order), but the actual IPC send is chained after any prior in-flight + * flush. This keeps chunks strictly ordered and lets `stopRecording`/ + * `cleanup` await the chain tail so an earlier timer-tick flush can't still + * be enqueuing chunks when `stop_dictation` fires. */ + const flushAudioBatch = useCallback((): Promise => { + const batch = audioBatchRef.current; + if (batch.length === 0) return flushChainRef.current; + + // Tag this flush with the session that owns the buffered audio. Captured + // once here so every chunk of this batch carries the same session; native + // drops chunks whose session no longer matches the active one, so a late + // flush from a just-stopped session can't leak into a newer session. + const session = nativeSessionRef.current; + + // Calculate total byte length and merge into a single buffer. Drain the + // batch synchronously so a concurrent timer tick can't grab the same + // samples — the merged buffer captured here is this flush's exclusive + // payload. + let totalSamples = 0; + for (const chunk of batch) { + totalSamples += chunk.length; + } + const merged = new Float32Array(totalSamples); + let offset = 0; + for (const chunk of batch) { + merged.set(chunk, offset); + offset += chunk.length; + } + audioBatchRef.current = []; + + // Split into chunks under the native IPC cap and send them in order. + // A single unbounded buffer can exceed the 100 KB native limit if the + // batch timer was delayed (e.g. main-thread stall), in which case native + // rejects it and the whole chunk is silently lost. + const sendChunks = async (): Promise => { + for (let start = 0; start < merged.length; start += MAX_IPC_SAMPLES) { + const slice = merged.subarray( + start, + Math.min(start + MAX_IPC_SAMPLES, merged.length), + ); + // Build the IPC payload: an 8-byte LE u64 session header followed by + // the audio bytes. Copy the slice into the header-prefixed buffer so + // the raw payload is exactly this chunk (a subarray view shares the + // parent's ArrayBuffer). + const payload = new Uint8Array(SESSION_HEADER_BYTES + slice.byteLength); + new DataView(payload.buffer).setBigUint64( + 0, + BigInt(session), + true, // little-endian + ); + payload.set( + new Uint8Array( + slice.buffer.slice( + slice.byteOffset, + slice.byteOffset + slice.byteLength, + ), + ), + SESSION_HEADER_BYTES, + ); + await invokeRawBinary("push_dictation_audio", payload).catch(() => {}); + } + }; + + // Chain this flush's IPC work after any prior in-flight flush so chunks + // stay strictly ordered and awaiting the tail drains everything. + const chained = flushChainRef.current.then(sendChunks); + flushChainRef.current = chained; + return chained; + }, []); + + const cleanup = useCallback(() => { + // Abort any in-flight startRecording so it bails after its next await + // instead of resuming and opening the mic/worklet or leaving the native + // session/listeners running after teardown. `cleanup` is the unmount + // handler (and the catch-path teardown); without this, unmounting while + // startRecording awaits `start_dictation`/`listen`/`getUserMedia`/ + // `addModule` would let the async start finish against a torn-down + // instance. A fresh startRecording clears this flag before its first + // await, so it never wrongly aborts a subsequent start. + startAbortedRef.current = true; + // Flush any remaining audio before teardown, then stop the native engine. + // Scope the stop to THIS hook instance's session. `cleanup` runs as every + // instance's unmount handler, so an unscoped stop here would let a + // non-recording composer (e.g. a thread reply composer closing) tear down + // the singleton engine owned by another composer that is actively + // recording. `nativeSessionRef.current` is 0 for an instance that never + // started a session (native IDs start at 1), so its stop can never match + // the live session and correctly no-ops. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + // Stop batch timer. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + // Disconnect audio worklet. Clear the port handler first so any PCM + // messages still queued on the main thread are dropped instead of + // appended to the (reused) audio batch. + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + // Drop any audio still buffered so it can't leak into the next session. + audioBatchRef.current = []; + // Close audio context. + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Unlisten events. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + }, [flushAudioBatch]); + + // Cleanup on unmount. + useEffect(() => cleanup, [cleanup]); + + const startRecording = useCallback(async () => { + // Also guard on `isTranscribing`: after `stopRecording()` the previous + // session has cleared `isRecording` but is still awaiting its native + // `stopped` event, which delivers the final transcript before its + // listeners are unregistered. Starting a new session in that window would + // unlisten the old session's handlers (below) before its last words + // arrived, dropping them. Wait for the prior session to fully stop. + if (!isEnabled || isStarting || isRecording || isTranscribing) return; + + // Clear abort flag for this new start attempt. + startAbortedRef.current = false; + // Reset any leftover audio buffer so a stale batch from a prior session + // can't be flushed into this new session/draft. + audioBatchRef.current = []; + // Reset the flush chain so this session's flushes don't chain behind a + // prior session's (already-settled) tail. + flushChainRef.current = Promise.resolve(); + + setIsStarting(true); + onRecordingStartRef.current?.(); + + try { + // 1. Start the native STT engine — returns the session ID used to tag events. + const sessionId = await invoke("start_dictation"); + nativeSessionRef.current = sessionId; + + // Bail if aborted during engine start. + if (startAbortedRef.current) { + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // Unregister any lingering listeners from a previous session so they + // can't match the new session ID via the shared ref. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + + // 2. Listen for transcript events from the native layer. + // Each listener captures `sessionId` by value (closure) so it only + // matches events from this specific session — immune to ref mutation. + const unlistenTranscript = await listen<{ + text: string; + session: number; + }>(DICTATION_TRANSCRIPT_EVENT, (event) => { + const { text, session } = event.payload; + if (session !== sessionId) return; + if (text) { + onTranscriptTextRef.current(text); + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenTranscriptRef.current = unlistenTranscript; + + const unlistenState = await listen<{ + state: string; + session: number; + }>(DICTATION_STATE_EVENT, (event) => { + const { state, session } = event.payload; + if (session !== sessionId) return; + if (state === "stopped") { + setIsRecording(false); + setIsTranscribing(false); + // Tear down this instance's local capture pipeline. The native + // engine is a singleton: when another mounted composer calls + // `start_dictation`, it stops this session's engine, so we can + // receive `stopped` without our own `stopRecording()`/`cleanup()` + // having run. Without tearing down here, `streamRef`/`workletRef`/ + // `batchTimerRef` stay alive and this composer keeps the mic open, + // pushing stale audio, until it unmounts. Don't re-invoke + // `stop_dictation` — the native side already stopped (that's why + // this event fired). + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + audioBatchRef.current = []; + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Clean up event listeners now that the session is fully done. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + unlistenState(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenStateRef.current = unlistenState; + + // 3. Capture mic audio. + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }); + streamRef.current = stream; + + // Bail if aborted during mic permission prompt. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // 4. Set up AudioWorklet to send PCM to native layer. + const audioContext = new AudioContext({ sampleRate: 48000 }); + audioContextRef.current = audioContext; + + // Resume if the WebView created the context suspended (autoplay policy). + // A suspended context never pulls the worklet's `process()`, so no PCM + // would reach `push_dictation_audio` while the UI shows an active + // recording. Mirrors the huddle capture path (`huddle/lib/audioWorklet.ts`). + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + // Create a processor that accumulates audio frames and posts them + // to the main thread. Batching happens on the main thread side via + // a timer to reduce IPC overhead. + const processorCode = ` + class DictationProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0] && input[0].length > 0) { + this.port.postMessage(input[0].buffer); + } + return true; + } + } + registerProcessor('dictation-processor', DictationProcessor); + `; + const blob = new Blob([processorCode], { + type: "application/javascript", + }); + const blobUrl = URL.createObjectURL(blob); + try { + await audioContext.audioWorklet.addModule(blobUrl); + } finally { + URL.revokeObjectURL(blobUrl); + } + + // Bail if stop/cancel was called while the worklet module was loading. + // Without this the worklet/flush timer would start and `isRecording` + // would be set true, leaving dictation running after it was stopped. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + void audioContext.close(); + audioContextRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + const source = audioContext.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode(audioContext, "dictation-processor"); + workletRef.current = worklet; + + // Accumulate audio frames in a batch array; a timer flushes to native. + worklet.port.onmessage = (event: MessageEvent) => { + audioBatchRef.current.push(new Float32Array(event.data)); + }; + + // Start the batch flush timer (~10 IPC calls/s instead of ~375). + batchTimerRef.current = setInterval(flushAudioBatch, AUDIO_BATCH_MS); + + const outputGain = audioContext.createGain(); + outputGain.gain.value = 0; + outputGainRef.current = outputGain; + + source.connect(worklet); + worklet.connect(outputGain); + outputGain.connect(audioContext.destination); + + setIsRecording(true); + setIsTranscribing(true); + } catch (error) { + // Tear down and stop the native engine if it was started but a later + // step failed (e.g. mic permission denied, AudioWorklet setup error). + // `cleanup()` performs the session-scoped stop, so it only tears down + // this instance's own session — never another composer's. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + + const message = + error instanceof Error ? error.message : "Local dictation failed"; + if (/not allowed|denied|permission/i.test(message)) { + toast.error("Microphone access denied", { + description: + "Allow microphone access in System Settings to use dictation.", + }); + } else if (/not found|no audio/i.test(message)) { + toast.error("No microphone found", { + description: "Connect a microphone and try again.", + }); + } else if (/model not ready/i.test(message)) { + toast.error("Voice model downloading", { + description: + "The speech model is still downloading. Try again shortly.", + }); + } else { + toast.error("Dictation failed", { description: message }); + } + } finally { + setIsStarting(false); + } + }, [ + cleanup, + flushAudioBatch, + isEnabled, + isRecording, + isStarting, + isTranscribing, + ]); + + const stopRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // Stop batch timer immediately. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic and audio pipeline immediately so the user gets visual feedback. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + // Clear the port handler so PCM messages still queued on the main + // thread are dropped rather than appended to the batch after the + // final flush below (and leaking into the next session). + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + setIsRecording(false); + // Flush remaining audio and THEN stop the native engine, ensuring the + // final batch arrives before the engine shuts down and flushes its buffer. + // isTranscribing stays true — cleared when `dictation-state: stopped` arrives. + // Scope the stop to THIS session: if the user restarts during the flush + // window, a new session may already be stored by the time this resolves — + // passing the session keeps this stop from tearing down the new engine. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + }, [flushAudioBatch]); + + const cancelRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // `cleanup()` performs the session-scoped native stop, so cancelling a + // composer that isn't the active recorder can't tear down another + // composer's session. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + }, [cleanup]); + + const toggleRecording = useCallback(() => { + if (isRecording || isStarting) { + stopRecording(); + return; + } + void startRecording(); + }, [isRecording, isStarting, startRecording, stopRecording]); + + return { + isEnabled, + isRecording, + isStarting, + isTranscribing, + startRecording, + stopRecording, + cancelRecording, + toggleRecording, + }; +} diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx new file mode 100644 index 0000000000..4f81656351 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -0,0 +1,49 @@ +import type * as React from "react"; +import { useRef } from "react"; +import { useFeatureEnabled } from "@/shared/features"; +import { DictationButton } from "../ui/DictationButton"; +import { useComposerDictation } from "./useComposerDictation"; + +interface UseMessageComposerDictationOptions { + syncContentRef: React.MutableRefObject<() => string>; + disabled: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + setComposerContent: (text: string) => void; + setEditorContent: (text: string) => void; + submitMessageRef: React.MutableRefObject<() => void>; + draftKey: string | null; + composerRef: React.RefObject; +} + +export function useMessageComposerDictation({ + setEditorContent, + ...options +}: UseMessageComposerDictationOptions) { + const enabled = useFeatureEnabled("voiceDictation"); + const setEditorContentRef = useRef(setEditorContent); + setEditorContentRef.current = setEditorContent; + return useComposerDictation({ + ...options, + enabled, + setEditorContentRef, + }); +} + +export function MessageComposerDictationAction({ + children, + dictation, + disabled, +}: { + children?: React.ReactNode; + dictation: ReturnType; + disabled: boolean; +}) { + return ( + <> + + {children} + + ); +} diff --git a/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts b/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts new file mode 100644 index 0000000000..31597ed7ae --- /dev/null +++ b/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts @@ -0,0 +1,53 @@ +import * as React from "react"; +import { useFeatureEnabled } from "@/shared/features"; +import { hasPrimaryShortcutModifier } from "@/shared/lib/platform"; + +export function useVoiceDictationShortcut(disabled: boolean) { + const enabled = useFeatureEnabled("voiceDictation"); + + React.useLayoutEffect(() => { + if (!enabled || disabled) { + return; + } + + let keyHeld = false; + const release = () => { + if (!keyHeld) return; + keyHeld = false; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-up")); + }; + const handleKeyDown = (event: KeyboardEvent) => { + if ( + event.defaultPrevented || + event.repeat || + event.altKey || + event.shiftKey || + event.key.toLowerCase() !== "d" || + !hasPrimaryShortcutModifier(event) + ) { + return; + } + event.preventDefault(); + keyHeld = true; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-down")); + }; + const handleKeyUp = (event: KeyboardEvent) => { + if (event.key.toLowerCase() === "d") release(); + }; + const handleVisibilityChange = () => { + if (document.hidden) release(); + }; + + window.addEventListener("keydown", handleKeyDown); + window.addEventListener("keyup", handleKeyUp); + window.addEventListener("blur", release); + document.addEventListener("visibilitychange", handleVisibilityChange); + return () => { + release(); + window.removeEventListener("keydown", handleKeyDown); + window.removeEventListener("keyup", handleKeyUp); + window.removeEventListener("blur", release); + document.removeEventListener("visibilitychange", handleVisibilityChange); + }; + }, [disabled, enabled]); +} diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts new file mode 100644 index 0000000000..12964f17af --- /dev/null +++ b/desktop/src/features/dictation/index.ts @@ -0,0 +1,8 @@ +export { useComposerDictation } from "./hooks/useComposerDictation"; +export { useDictation } from "./hooks/useDictation"; +export { + MessageComposerDictationAction, + useMessageComposerDictation, +} from "./hooks/useMessageComposerDictation"; +export { useVoiceDictationShortcut } from "./hooks/useVoiceDictationShortcut"; +export { DictationButton } from "./ui/DictationButton"; diff --git a/desktop/src/features/dictation/lib/activeComposer.ts b/desktop/src/features/dictation/lib/activeComposer.ts new file mode 100644 index 0000000000..1fb1b3f104 --- /dev/null +++ b/desktop/src/features/dictation/lib/activeComposer.ts @@ -0,0 +1,23 @@ +/** + * Tracks which composer instance should handle global dictation shortcuts. + * + * When multiple composers are mounted (e.g. channel + thread), only the + * most recently focused one should respond to ⌘D. Each composer registers + * on focus and the global shortcut only dispatches to the active instance. + */ + +let activeInstanceId: string | null = null; + +export function setActiveDictationComposer(id: string): void { + activeInstanceId = id; +} + +export function clearActiveDictationComposer(id: string): void { + if (activeInstanceId === id) { + activeInstanceId = null; + } +} + +export function isActiveDictationComposer(id: string): boolean { + return activeInstanceId === id; +} diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs new file mode 100644 index 0000000000..151104cbd8 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -0,0 +1,150 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "./voiceInput.ts"; + +// ── parseAutoSubmitPhrases ────────────────────────────────────────────────── + +test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => { + assert.deepEqual(parseAutoSubmitPhrases(null), []); + assert.deepEqual(parseAutoSubmitPhrases(undefined), []); + assert.deepEqual(parseAutoSubmitPhrases(""), []); +}); + +test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => { + assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [ + "submit", + "send it", + ]); +}); + +test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => { + assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]); +}); + +// ── replaceTrailingTranscribedText ────────────────────────────────────────── + +test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", "world"), + "Hello world", + ); +}); + +test("replaceTrailingTranscribedText_appendsToEmptyBase", () => { + assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello"); +}); + +test("replaceTrailingTranscribedText_replacesTrailingInterim", () => { + // Interim "hello wor" is refined to "hello world". + assert.equal( + replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"), + "hello world", + ); +}); + +test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => { + // User typed "Note: " then dictated; the manual prefix must survive. + assert.equal( + replaceTrailingTranscribedText("Note: hi", "hi", "hi there"), + "Note: hi there", + ); +}); + +test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => { + // If the previous transcript isn't the trailing text anymore, append. + assert.equal( + replaceTrailingTranscribedText("edited text", "old", "new"), + "edited text new", + ); +}); + +test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", ", world"), + "Hello, world", + ); +}); + +// ── getAutoSubmitMatch ────────────────────────────────────────────────────── + +test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => { + assert.equal( + getAutoSubmitMatch("hello there", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_returnsNullWhenPhrasesEmpty", () => { + // DEFAULT_AUTO_SUBMIT_PHRASE is empty (auto-submit disabled by default). + assert.equal( + getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ), + null, + ); +}); + +test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => { + const match = getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit"); + assert.equal(match.textWithoutPhrase, "send this message"); +}); + +test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => { + // "submit" is not at the end, so it must not auto-send. + assert.equal( + getAutoSubmitMatch( + "submit the form later", + parseAutoSubmitPhrases("submit"), + ), + null, + ); +}); + +test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => { + // "resubmit" ends with "submit" but is not a standalone word → no match. + assert.equal( + getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => { + const match = getAutoSubmitMatch( + "ship it submit.", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.textWithoutPhrase, "ship it"); +}); + +test("getAutoSubmitMatch_matchesMultiWordPhrase", () => { + const match = getAutoSubmitMatch( + "please do this send it", + parseAutoSubmitPhrases("send it"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "send it"); + assert.equal(match.textWithoutPhrase, "please do this"); +}); + +test("getAutoSubmitMatch_prefersLongestPhrase", () => { + const match = getAutoSubmitMatch( + "text please submit now", + parseAutoSubmitPhrases("submit now, now"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit now"); + assert.equal(match.textWithoutPhrase, "text please"); +}); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts new file mode 100644 index 0000000000..1c287cef72 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -0,0 +1,114 @@ +/** + * Default auto-submit phrase. Empty string disables auto-submit — the user + * must manually press Enter/Send after dictation. The infrastructure for + * configurable phrases is in place (parseAutoSubmitPhrases, getAutoSubmitMatch) + * and can be wired to a user setting when we're ready to ship auto-submit. + */ +export const DEFAULT_AUTO_SUBMIT_PHRASE = ""; + +const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function normalizePhrase(value: string): string { + return value + .toLowerCase() + .replace(/\s+/g, " ") + .trim() + .replace(TRAILING_PUNCTUATION_REGEX, "") + .trim(); +} + +export function parseAutoSubmitPhrases( + rawValue: string | null | undefined, +): string[] { + if (!rawValue) return []; + return Array.from( + new Set( + rawValue + .split(",") + .map((value) => normalizePhrase(value)) + .filter(Boolean), + ), + ); +} + +function appendTranscribedText(baseText: string, fragment: string): string { + const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); + if (!normalizedFragment) return baseText; + if (!baseText.trim()) return normalizedFragment; + if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) { + return `${baseText}${normalizedFragment}`; + } + return `${baseText} ${normalizedFragment}`; +} + +export function replaceTrailingTranscribedText( + fullText: string, + previousTranscribedText: string, + nextTranscribedText: string, +): string { + if (!previousTranscribedText) { + return appendTranscribedText(fullText, nextTranscribedText); + } + + if (fullText.endsWith(previousTranscribedText)) { + return appendTranscribedText( + fullText.slice(0, -previousTranscribedText.length), + nextTranscribedText, + ); + } + + const trimmedPreviousText = previousTranscribedText.trim(); + if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) { + return appendTranscribedText( + fullText.slice(0, -trimmedPreviousText.length), + nextTranscribedText, + ); + } + + return appendTranscribedText(fullText, nextTranscribedText); +} + +export function getAutoSubmitMatch( + transcribedText: string, + autoSubmitPhrases: string[], +): { matchedPhrase: string; textWithoutPhrase: string } | null { + const normalizedTranscribedText = normalizePhrase(transcribedText); + if (!normalizedTranscribedText) return null; + + const sortedPhrases = [...autoSubmitPhrases].sort( + (left, right) => right.length - left.length, + ); + + for (const phrase of sortedPhrases) { + if (!normalizedTranscribedText.endsWith(phrase)) continue; + + const phraseStartIndex = normalizedTranscribedText.length - phrase.length; + if ( + phraseStartIndex > 0 && + normalizedTranscribedText[phraseStartIndex - 1] !== " " + ) { + continue; + } + + const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, ""); + const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp); + const phrasePattern = new RegExp( + `(^|\\s)(${phraseWords.join("\\s+")})\\s*$`, + "iu", + ); + const rawMatch = trimmedText.match(phrasePattern); + const phraseStartOffset = + rawMatch && rawMatch.index !== undefined + ? rawMatch.index + (rawMatch[1]?.length ?? 0) + : trimmedText.length - phrase.length; + const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd(); + + return { matchedPhrase: phrase, textWithoutPhrase }; + } + + return null; +} diff --git a/desktop/src/features/dictation/ui/DictationButton.tsx b/desktop/src/features/dictation/ui/DictationButton.tsx new file mode 100644 index 0000000000..f284957e17 --- /dev/null +++ b/desktop/src/features/dictation/ui/DictationButton.tsx @@ -0,0 +1,73 @@ +import { Mic, Square } from "lucide-react"; +import { Button } from "@/shared/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; +import { cn } from "@/shared/lib/cn"; +import { isMacPlatform } from "@/shared/lib/platform"; + +interface DictationState { + isEnabled: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; + toggleRecording: () => void; +} + +interface DictationButtonProps { + dictation: DictationState; + disabled?: boolean; +} + +export function DictationButton({ + dictation, + disabled = false, +}: DictationButtonProps) { + if (!dictation.isEnabled) return null; + + const shortcutHint = isMacPlatform() ? "⌘D" : "Ctrl+D"; + + const tooltipText = dictation.isRecording + ? "Stop recording" + : dictation.isTranscribing + ? "Transcribing…" + : null; + + return ( + + + + + + {tooltipText ?? ( + + Voice Dictation + + {shortcutHint} + + + )} + + + ); +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 69e4ec67b5..ef040c68dd 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -2,9 +2,9 @@ import * as React from "react"; import { EditorContent } from "@tiptap/react"; import { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; +import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import { handleAgentSnapshotPaste } from "@/features/messages/lib/agentSnapshotClipboard"; import { useComposerAutofocus } from "@/features/messages/lib/useComposerAutofocus"; -import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import { useDrafts } from "@/features/messages/lib/useDrafts"; import { resolveSentDraftKey } from "@/features/messages/ui/draftSubmitKey"; import { useEmojiAutocomplete } from "@/features/messages/lib/useEmojiAutocomplete"; @@ -39,22 +39,26 @@ import { import { useLinkEditor } from "@/features/messages/lib/useLinkEditor"; import { useComposerSpoilerParticles } from "@/features/messages/lib/useComposerSpoilerParticles"; import { useTypingBroadcast } from "@/features/messages/useTypingBroadcast"; +import { + MessageComposerDictationAction, + useMessageComposerDictation, +} from "@/features/dictation"; import { getBuzzCodeBlockClipboardText } from "@/shared/lib/codeBlockClipboard"; import { cn } from "@/shared/lib/cn"; -import { ChannelAutocomplete } from "./ChannelAutocomplete"; import { ComposerReplyEditBanner } from "./ComposerReplyEditBanner"; import { ComposerAttachments, DropZoneOverlay } from "./ComposerAttachments"; -import { EmojiAutocomplete } from "./EmojiAutocomplete"; -import { - MentionAutocomplete, - type MentionSuggestion, -} from "./MentionAutocomplete"; +import type { MentionSuggestion } from "./MentionAutocomplete"; import { ComposerDockToolbar } from "./ComposerDockToolbar"; import { NonMemberMentionDialog } from "./NonMemberMentionDialog"; import { useMentionSendFlow } from "./useMentionSendFlow"; import { usePersistentAgentMentionHydration } from "./usePersistentAgentMentionHydration"; import { useComposerContentState } from "./useComposerContentState"; import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; +import { + MessageComposerAutocompletes, + MessageComposerUploadError, +} from "./MessageComposerOverlays"; +import { useComposerScrollToBottom } from "./useComposerScrollToBottom"; import type { MessageComposerProps } from "./MessageComposer.types"; @@ -206,6 +210,7 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); + const stopDictationRef = React.useRef<() => void>(() => {}); const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler @@ -219,13 +224,7 @@ function MessageComposerImpl({ >(null); const onLinkShortcutRef = React.useRef<(() => boolean) | null>(null); - const scrollComposerToBottom = React.useCallback(() => { - window.requestAnimationFrame(() => { - const scrollElement = composerScrollRef.current; - if (!scrollElement) return; - scrollElement.scrollTop = scrollElement.scrollHeight; - }); - }, []); + const scrollComposerToBottom = useComposerScrollToBottom(composerScrollRef); const computedPlaceholder = editTarget ? "Edit your message" @@ -268,6 +267,19 @@ function MessageComposerImpl({ }, }); + const dictation = useMessageComposerDictation({ + syncContentRef: syncContentRefFromEditorRef, + disabled, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContent: richText.setContent, + submitMessageRef, + draftKey: effectiveDraftKey, + composerRef: composerScrollRef, + }); + stopDictationRef.current = dictation.cancelRecording; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); @@ -278,7 +290,6 @@ function MessageComposerImpl({ onLinkSelectionChangeRef.current = linkEditor.showFromCursor; onLinkShortcutRef.current = linkEditor.openFromShortcut; useComposerSpoilerParticles(richText.editor, composerScrollRef); - const persistentMentionHydration = usePersistentAgentMentionHydration({ audienceScope, hydrationKey: effectiveDraftKey, @@ -512,6 +523,7 @@ function MessageComposerImpl({ // Edit mode if (editTargetRef.current && onEditSaveRef.current) { if (isSendingRef.current || isUploadingRef.current) return; + stopDictationRef.current(); const currentPendingImeta = media.pendingImetaRef.current; // No empty-edit guard here: clearing an edit to empty (no text, no // attachments) flows through to onEditSave as empty content, which @@ -587,6 +599,7 @@ function MessageComposerImpl({ ) { return; } + stopDictationRef.current(); const capturedThreadContext = onCaptureSendContext?.() ?? null; if ( @@ -919,42 +932,22 @@ function MessageComposerImpl({ }} > {ownsDropZone && media.isDragOver && } - - media.setUploadState({ status: "idle" })} /> - - {media.uploadState.status === "error" ? ( -
- Upload failed: {media.uploadState.message} - -
- ) : null} {(media.pendingImeta.length > 0 || media.isUploading) && (
@@ -988,7 +981,14 @@ function MessageComposerImpl({ layoutMode={layoutMode} composerDisabled={disabled} editor={richText.editor} - extraActions={toolbarExtraActions} + extraActions={ + + {toolbarExtraActions} + + } formattingDisabled={disabled} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen} diff --git a/desktop/src/features/messages/ui/MessageComposerOverlays.tsx b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx new file mode 100644 index 0000000000..67b4f030b3 --- /dev/null +++ b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx @@ -0,0 +1,72 @@ +import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; +import type { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; +import type { EmojiSuggestion } from "@/features/messages/lib/useEmojiAutocomplete"; +import type { useEmojiAutocomplete } from "@/features/messages/lib/useEmojiAutocomplete"; +import type { useMentions } from "@/features/messages/lib/useMentions"; +import { ChannelAutocomplete } from "./ChannelAutocomplete"; +import { EmojiAutocomplete } from "./EmojiAutocomplete"; +import { + MentionAutocomplete, + type MentionSuggestion, +} from "./MentionAutocomplete"; + +export function MessageComposerAutocompletes({ + applyChannelInsert, + applyEmojiInsert, + applyMentionInsert, + channelLinks, + emojiAutocomplete, + mentions, +}: { + applyChannelInsert: (suggestion: ChannelSuggestion) => void; + applyEmojiInsert: (suggestion: EmojiSuggestion) => void; + applyMentionInsert: (suggestion: MentionSuggestion) => void; + channelLinks: ReturnType; + emojiAutocomplete: ReturnType; + mentions: ReturnType; +}) { + return ( + <> + + + + + ); +} + +export function MessageComposerUploadError({ + message, + onDismiss, +}: { + message: string | null | undefined; + onDismiss: () => void; +}) { + if (message == null) return null; + return ( +
+ Upload failed: {message} + +
+ ); +} diff --git a/desktop/src/features/messages/ui/useComposerScrollToBottom.ts b/desktop/src/features/messages/ui/useComposerScrollToBottom.ts new file mode 100644 index 0000000000..9d07efb3a5 --- /dev/null +++ b/desktop/src/features/messages/ui/useComposerScrollToBottom.ts @@ -0,0 +1,13 @@ +import * as React from "react"; + +export function useComposerScrollToBottom( + composerScrollRef: React.RefObject, +) { + return React.useCallback(() => { + window.requestAnimationFrame(() => { + const scrollElement = composerScrollRef.current; + if (!scrollElement) return; + scrollElement.scrollTop = scrollElement.scrollHeight; + }); + }, [composerScrollRef]); +} diff --git a/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx b/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx index c71e62d5d9..af7bfafbb3 100644 --- a/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx +++ b/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx @@ -3,6 +3,7 @@ import { getPlatformKeys, type KeyboardShortcut, } from "@/shared/lib/keyboard-shortcuts"; +import { useFeatureEnabled } from "@/shared/features"; import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; @@ -29,6 +30,7 @@ function KeyCombo({ shortcut }: { shortcut: KeyboardShortcut }) { } export function KeyboardShortcutsCard() { + const voiceDictationEnabled = useFeatureEnabled("voiceDictation"); const categories = getShortcutsByCategory(); return ( @@ -45,22 +47,27 @@ export function KeyboardShortcutsCard() { {category} - {shortcuts.map((shortcut) => ( - -
- - {shortcut.label} - - - {shortcut.description} - -
- -
- ))} + {shortcuts + .filter( + (shortcut) => + voiceDictationEnabled || shortcut.id !== "voice-dictation", + ) + .map((shortcut) => ( + +
+ + {shortcut.label} + + + {shortcut.description} + +
+ +
+ ))}
))} diff --git a/desktop/src/shared/lib/keyboard-shortcuts.ts b/desktop/src/shared/lib/keyboard-shortcuts.ts index 422c45c5f3..377e4ae616 100644 --- a/desktop/src/shared/lib/keyboard-shortcuts.ts +++ b/desktop/src/shared/lib/keyboard-shortcuts.ts @@ -181,6 +181,14 @@ export const KEYBOARD_SHORTCUTS: KeyboardShortcut[] = [ keysWindows: "Ctrl+Space", category: "Messages", }, + { + id: "voice-dictation", + label: "Voice Dictation", + description: "Hold to dictate, release to stop (push-to-talk)", + keys: "⌘D", + keysWindows: "Ctrl+D", + category: "Messages", + }, // Formatting { diff --git a/desktop/tests/e2e/messaging.spec.ts b/desktop/tests/e2e/messaging.spec.ts index c6f5aefb9b..3e6a0ae40c 100644 --- a/desktop/tests/e2e/messaging.spec.ts +++ b/desktop/tests/e2e/messaging.spec.ts @@ -109,7 +109,72 @@ test.beforeEach(async ({ page }, testInfo) => { ], } : undefined; - await installMockBridge(page, mock); + await installMockBridge(page, mock, { + seedPreviewFeatures: !testInfo.title.includes("dictation experiment"), + }); +}); + +test("voice dictation stays behind the dictation experiment", async ({ + page, +}) => { + await page.goto("/"); + await page.getByTestId("channel-general").click(); + + await page.evaluate(() => { + let dictationStarts = 0; + window.addEventListener("buzz:dictation-key-down", () => { + dictationStarts += 1; + }); + ( + window as typeof window & { __BUZZ_DICTATION_STARTS__?: () => number } + ).__BUZZ_DICTATION_STARTS__ = () => dictationStarts; + }); + + const dispatchDictationShortcut = () => + page.evaluate(() => { + const isMac = /mac|iphone|ipad|ipod/i.test(navigator.platform); + window.dispatchEvent( + new KeyboardEvent("keydown", { + bubbles: true, + ctrlKey: !isMac, + key: "d", + metaKey: isMac, + }), + ); + window.dispatchEvent( + new KeyboardEvent("keyup", { + bubbles: true, + ctrlKey: !isMac, + key: "d", + metaKey: isMac, + }), + ); + }); + const getDictationStarts = () => + page.evaluate( + () => + ( + window as typeof window & { + __BUZZ_DICTATION_STARTS__?: () => number; + } + ).__BUZZ_DICTATION_STARTS__?.() ?? 0, + ); + + await page.getByTestId("message-input").click(); + await dispatchDictationShortcut(); + await expect.poll(getDictationStarts).toBe(0); + + await openSettings(page, "experimental"); + const dictationToggle = page.getByTestId("feature-toggle-voiceDictation"); + await expect(dictationToggle).not.toBeChecked(); + await dictationToggle.click(); + await expect(dictationToggle).toBeChecked(); + + await page.getByTestId("settings-back-to-app").click(); + await expect(page.getByTestId("message-input")).toBeVisible(); + await page.getByTestId("message-input").click(); + await dispatchDictationShortcut(); + await expect.poll(getDictationStarts).toBe(1); }); test("agent owner label identifies the agent and owner", async ({ page }) => { diff --git a/preview-features.json b/preview-features.json index 388f1c39b0..cfd37ee873 100644 --- a/preview-features.json +++ b/preview-features.json @@ -30,6 +30,12 @@ "name": "Agent-managed profiles", "description": "Let agents manage their own relay name and avatar instead of restoring the desktop copy", "platforms": ["desktop"] + }, + { + "id": "voiceDictation", + "name": "Voice dictation", + "description": "Compose messages with local, real-time speech recognition", + "platforms": ["desktop"] } ] } From 30ba73e289802936504994a757a8a378c7f8860d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 11:41:49 -0400 Subject: [PATCH 03/13] fix(desktop): preserve final dictated speech Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- desktop/src-tauri/src/huddle/stt.rs | 106 ++++++++++++-- .../dictation/hooks/useComposerDictation.ts | 51 ++++--- .../features/dictation/hooks/useDictation.ts | 55 +------- .../hooks/useMessageComposerDictation.tsx | 24 +++- desktop/src/features/dictation/index.ts | 1 + .../dictation/lib/voiceInput.test.mjs | 131 +++++------------- .../src/features/dictation/lib/voiceInput.ts | 94 +++---------- .../features/messages/ui/MessageComposer.tsx | 12 +- 8 files changed, 205 insertions(+), 269 deletions(-) diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 67b639332a..a66c63ae75 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -220,6 +220,34 @@ const VAD_THRESHOLD: f32 = 0.5; /// How long the worker waits on the audio channel before checking the shutdown flag. const RECV_TIMEOUT: Duration = Duration::from_millis(50); +enum AudioReceive { + Bytes(Vec), + Retry, + Stop, +} + +fn receive_audio( + audio_rx: &Receiver>, + is_shutting_down: bool, + flush_on_shutdown: bool, +) -> AudioReceive { + if is_shutting_down { + if !flush_on_shutdown { + return AudioReceive::Stop; + } + return match audio_rx.try_recv() { + Ok(bytes) => AudioReceive::Bytes(bytes), + Err(mpsc::TryRecvError::Empty | mpsc::TryRecvError::Disconnected) => AudioReceive::Stop, + }; + } + + match audio_rx.recv_timeout(RECV_TIMEOUT) { + Ok(bytes) => AudioReceive::Bytes(bytes), + Err(mpsc::RecvTimeoutError::Timeout) => AudioReceive::Retry, + Err(mpsc::RecvTimeoutError::Disconnected) => AudioReceive::Stop, + } +} + /// 50 ms cooldown after TTS stops before STT re-enables. /// Prevents the tail of TTS audio from being transcribed as speech. /// Previous value (200 ms) was eating the first word when the user spoke @@ -319,10 +347,7 @@ fn stt_worker( .as_ref() .is_some_and(|p| p.load(Ordering::Acquire)); loop { - // Check shutdown flag before blocking. - if shutdown.load(Ordering::Acquire) { - break; - } + let is_shutting_down = shutdown.load(Ordering::Acquire); // Track TTS transitions to set the cooldown timer. let tts_now = config.tts_active.load(Ordering::Acquire); @@ -347,11 +372,13 @@ fn stt_worker( ptt_was_active = ptt_now; } - // Use recv_timeout so we can periodically check the shutdown flag. - let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { - Ok(b) => b, - Err(mpsc::RecvTimeoutError::Timeout) => continue, - Err(mpsc::RecvTimeoutError::Disconnected) => break, // Sender dropped. + // Dictation drains audio already accepted by the IPC command before + // flushing its final transcript. Huddles retain their immediate + // shutdown behavior so leaving cannot publish a late transcript. + let bytes = match receive_audio(&audio_rx, is_shutting_down, config.flush_on_shutdown) { + AudioReceive::Bytes(bytes) => bytes, + AudioReceive::Retry => continue, + AudioReceive::Stop => break, }; // Drain any additional pending messages to batch-process. @@ -391,8 +418,32 @@ fn stt_worker( } } - if config.flush_on_shutdown && !speech_buf.is_empty() { - flush_to_stt(&speech_buf, &recognizer, &text_tx); + if config.flush_on_shutdown { + if !input_buf_48k.is_empty() { + input_buf_48k.resize(chunk_in, 0.0); + let resampled = resample_chunk(&mut resampler, &input_buf_48k); + process_16k_samples( + &resampled, + &mut leftover_16k, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + &config.tts_active, + config.tts_cancel.as_deref(), + &mut tts_stopped_at, + config.ptt_active.as_ref(), + config.silence_flush_frames, + config.max_speech_samples, + config.partial_flush_samples, + ); + } + if !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + } } } @@ -607,3 +658,36 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. use super::drain_until_shutdown; + +#[cfg(test)] +mod tests { + use std::sync::mpsc; + + use super::{receive_audio, AudioReceive}; + + #[test] + fn dictation_shutdown_drains_accepted_audio() { + let (audio_tx, audio_rx) = mpsc::sync_channel(1); + audio_tx.send(vec![1, 2, 3, 4]).unwrap(); + + assert!(matches!( + receive_audio(&audio_rx, true, true), + AudioReceive::Bytes(bytes) if bytes == [1, 2, 3, 4] + )); + assert!(matches!( + receive_audio(&audio_rx, true, true), + AudioReceive::Stop + )); + } + + #[test] + fn huddle_shutdown_does_not_drain_accepted_audio() { + let (audio_tx, audio_rx) = mpsc::sync_channel(1); + audio_tx.send(vec![1, 2, 3, 4]).unwrap(); + + assert!(matches!( + receive_audio(&audio_rx, true, false), + AudioReceive::Stop + )); + } +} diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts index d11b2b3042..b8869f8033 100644 --- a/desktop/src/features/dictation/hooks/useComposerDictation.ts +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -21,7 +21,6 @@ interface UseComposerDictationOptions { setComposerContent: (text: string) => void; /** Ref to a function that updates the Tiptap editor document. */ setEditorContentRef: React.MutableRefObject<(text: string) => void>; - submitMessageRef: React.MutableRefObject<() => void>; /** When this key changes (channel/thread switch), active dictation is stopped. */ draftKey?: string | null; /** Ref to the composer's container element for focus tracking. */ @@ -30,7 +29,7 @@ interface UseComposerDictationOptions { /** * Thin wrapper around `useDictation` pre-wired for the MessageComposer's - * state management (syncContentRef, setComposerContent, editor, submitMessageRef). + * state management (syncContentRef, setComposerContent, editor). * * Uses the local Parakeet STT engine — fully offline, no relay or API key needed. */ @@ -43,7 +42,6 @@ export function useComposerDictation({ isUploadingRef, setComposerContent, setEditorContentRef, - submitMessageRef, draftKey, composerRef, }: UseComposerDictationOptions) { @@ -62,16 +60,16 @@ export function useComposerDictation({ setComposerContent(text); setEditorContentRef.current(text); }, - onSend: (text) => { - setComposerContent(text); - setEditorContentRef.current(text); - // Submit synchronously — the content ref is already set above, so - // syncComposerContentFromEditor() will serialize the editor which now - // holds the dictated text. - submitMessageRef.current(); - }, - isSendBlockedRef, }); + const startRecordingRef = useRef(dictation.startRecording); + const stopRecordingRef = useRef(dictation.stopRecording); + const isRecordingRef = useRef(dictation.isRecording); + const isStartingRef = useRef(dictation.isStarting); + const shortcutHeldRef = useRef(false); + startRecordingRef.current = dictation.startRecording; + stopRecordingRef.current = dictation.stopRecording; + isRecordingRef.current = dictation.isRecording; + isStartingRef.current = dictation.isStarting; // Track which composer is active (most recently focused) so that the global // ⌘D shortcut only dispatches to one instance when multiple are mounted. @@ -162,30 +160,31 @@ export function useComposerDictation({ if (el && !el.contains(document.activeElement)) return; // Don't start dictation in disabled/blocked composers. if (disabledRef.current || isSendBlockedRef.current) return; - if (!dictation.isRecording && !dictation.isStarting) { - dictation.startRecording(); + if ( + !shortcutHeldRef.current && + !isRecordingRef.current && + !isStartingRef.current + ) { + shortcutHeldRef.current = true; + void startRecordingRef.current(); } } function handleKeyUp() { - if (dictation.isRecording || dictation.isStarting) { - dictation.stopRecording(); - } + if (!shortcutHeldRef.current) return; + shortcutHeldRef.current = false; + stopRecordingRef.current(); } window.addEventListener("buzz:dictation-key-down", handleKeyDown); window.addEventListener("buzz:dictation-key-up", handleKeyUp); return () => { window.removeEventListener("buzz:dictation-key-down", handleKeyDown); window.removeEventListener("buzz:dictation-key-up", handleKeyUp); + if (shortcutHeldRef.current) { + shortcutHeldRef.current = false; + stopRecordingRef.current(); + } }; - }, [ - instanceId, - composerRef, - enabled, - dictation.isRecording, - dictation.isStarting, - dictation.startRecording, - dictation.stopRecording, - ]); + }, [instanceId, composerRef, enabled]); return dictation; } diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts index c11fc4b3be..34a8bf2453 100644 --- a/desktop/src/features/dictation/hooks/useDictation.ts +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -1,11 +1,5 @@ -import type * as React from "react"; -import { useCallback, useMemo, useRef } from "react"; -import { - DEFAULT_AUTO_SUBMIT_PHRASE, - getAutoSubmitMatch, - parseAutoSubmitPhrases, - replaceTrailingTranscribedText, -} from "../lib/voiceInput"; +import { useCallback, useRef } from "react"; +import { replaceTrailingTranscribedText } from "../lib/voiceInput"; import { useLocalDictation } from "./useLocalDictation"; interface UseDictationOptions { @@ -15,24 +9,13 @@ interface UseDictationOptions { getText: () => string; /** Set composer text */ setText: (value: string) => void; - /** Send the message */ - onSend: (text: string) => void; - /** Ref that is `true` when sending is blocked (uploading, preparing mention, etc.) */ - isSendBlockedRef?: React.MutableRefObject; } export function useDictation({ disabled = false, getText, setText, - onSend, - isSendBlockedRef, }: UseDictationOptions) { - const autoSubmitPhrases = useMemo( - () => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), - [], - ); - const stopRecordingRef = useRef<() => void>(() => {}); const lastTranscriptRef = useRef(""); const handleTranscript = useCallback( @@ -44,36 +27,12 @@ export function useDictation({ previous, transcript, ); - const match = getAutoSubmitMatch(transcript, autoSubmitPhrases); - - if (!match) { - setText(merged); - // Reset to empty — each streaming partial is an independent segment - // (the native engine flushes and clears its buffer). The next transcript - // should be appended, not replace this one. - lastTranscriptRef.current = ""; - return; - } - - const textWithoutPhrase = replaceTrailingTranscribedText( - latest, - previous, - match.textWithoutPhrase, - ); - if (!textWithoutPhrase.trim()) return; - - stopRecordingRef.current(); - - if (isSendBlockedRef?.current) { - setText(textWithoutPhrase); - return; - } - - setText(textWithoutPhrase.trim()); - onSend(textWithoutPhrase.trim()); + setText(merged); + // Each native flush is an independent segment, so the next transcript + // appends instead of replacing this one. lastTranscriptRef.current = ""; }, - [autoSubmitPhrases, getText, onSend, isSendBlockedRef, setText], + [getText, setText], ); const dictation = useLocalDictation({ @@ -84,7 +43,5 @@ export function useDictation({ onTranscriptText: handleTranscript, }); - stopRecordingRef.current = dictation.stopRecording; - return dictation; } diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx index 4f81656351..407d6d9a1e 100644 --- a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -1,6 +1,7 @@ import type * as React from "react"; -import { useRef } from "react"; +import { useCallback, useRef } from "react"; import { useFeatureEnabled } from "@/shared/features"; +import { getDictationSendDecision } from "../lib/voiceInput"; import { DictationButton } from "../ui/DictationButton"; import { useComposerDictation } from "./useComposerDictation"; @@ -12,7 +13,6 @@ interface UseMessageComposerDictationOptions { isUploadingRef: React.MutableRefObject; setComposerContent: (text: string) => void; setEditorContent: (text: string) => void; - submitMessageRef: React.MutableRefObject<() => void>; draftKey: string | null; composerRef: React.RefObject; } @@ -24,11 +24,29 @@ export function useMessageComposerDictation({ const enabled = useFeatureEnabled("voiceDictation"); const setEditorContentRef = useRef(setEditorContent); setEditorContentRef.current = setEditorContent; - return useComposerDictation({ + const dictation = useComposerDictation({ ...options, enabled, setEditorContentRef, }); + const { isRecording, isStarting, isTranscribing, stopRecording } = dictation; + const prepareToSubmit = useCallback(() => { + const decision = getDictationSendDecision({ + isRecording, + isStarting, + isTranscribing, + }); + if (decision === "stop-recording") { + stopRecording(); + } + return decision === "send"; + }, [isRecording, isStarting, isTranscribing, stopRecording]); + + return { + ...dictation, + isSendBlocked: isRecording || isStarting || isTranscribing, + prepareToSubmit, + }; } export function MessageComposerDictationAction({ diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts index 12964f17af..502b37e6b9 100644 --- a/desktop/src/features/dictation/index.ts +++ b/desktop/src/features/dictation/index.ts @@ -6,3 +6,4 @@ export { } from "./hooks/useMessageComposerDictation"; export { useVoiceDictationShortcut } from "./hooks/useVoiceDictationShortcut"; export { DictationButton } from "./ui/DictationButton"; +export { getDictationSendDecision } from "./lib/voiceInput"; diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs index 151104cbd8..908a7bd7f8 100644 --- a/desktop/src/features/dictation/lib/voiceInput.test.mjs +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -2,29 +2,46 @@ import assert from "node:assert/strict"; import test from "node:test"; import { - DEFAULT_AUTO_SUBMIT_PHRASE, - getAutoSubmitMatch, - parseAutoSubmitPhrases, + getDictationSendDecision, replaceTrailingTranscribedText, } from "./voiceInput.ts"; -// ── parseAutoSubmitPhrases ────────────────────────────────────────────────── - -test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => { - assert.deepEqual(parseAutoSubmitPhrases(null), []); - assert.deepEqual(parseAutoSubmitPhrases(undefined), []); - assert.deepEqual(parseAutoSubmitPhrases(""), []); -}); - -test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => { - assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [ - "submit", - "send it", - ]); +test("dictation send stops capture before submitting", () => { + assert.equal( + getDictationSendDecision({ + isRecording: true, + isStarting: false, + isTranscribing: true, + }), + "stop-recording", + ); + assert.equal( + getDictationSendDecision({ + isRecording: false, + isStarting: true, + isTranscribing: false, + }), + "stop-recording", + ); }); -test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => { - assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]); +test("dictation send waits for the final transcript", () => { + assert.equal( + getDictationSendDecision({ + isRecording: false, + isStarting: false, + isTranscribing: true, + }), + "wait", + ); + assert.equal( + getDictationSendDecision({ + isRecording: false, + isStarting: false, + isTranscribing: false, + }), + "send", + ); }); // ── replaceTrailingTranscribedText ────────────────────────────────────────── @@ -70,81 +87,3 @@ test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { "Hello, world", ); }); - -// ── getAutoSubmitMatch ────────────────────────────────────────────────────── - -test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => { - assert.equal( - getAutoSubmitMatch("hello there", parseAutoSubmitPhrases("submit")), - null, - ); -}); - -test("getAutoSubmitMatch_returnsNullWhenPhrasesEmpty", () => { - // DEFAULT_AUTO_SUBMIT_PHRASE is empty (auto-submit disabled by default). - assert.equal( - getAutoSubmitMatch( - "send this message submit", - parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), - ), - null, - ); -}); - -test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => { - const match = getAutoSubmitMatch( - "send this message submit", - parseAutoSubmitPhrases("submit"), - ); - assert.ok(match); - assert.equal(match.matchedPhrase, "submit"); - assert.equal(match.textWithoutPhrase, "send this message"); -}); - -test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => { - // "submit" is not at the end, so it must not auto-send. - assert.equal( - getAutoSubmitMatch( - "submit the form later", - parseAutoSubmitPhrases("submit"), - ), - null, - ); -}); - -test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => { - // "resubmit" ends with "submit" but is not a standalone word → no match. - assert.equal( - getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")), - null, - ); -}); - -test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => { - const match = getAutoSubmitMatch( - "ship it submit.", - parseAutoSubmitPhrases("submit"), - ); - assert.ok(match); - assert.equal(match.textWithoutPhrase, "ship it"); -}); - -test("getAutoSubmitMatch_matchesMultiWordPhrase", () => { - const match = getAutoSubmitMatch( - "please do this send it", - parseAutoSubmitPhrases("send it"), - ); - assert.ok(match); - assert.equal(match.matchedPhrase, "send it"); - assert.equal(match.textWithoutPhrase, "please do this"); -}); - -test("getAutoSubmitMatch_prefersLongestPhrase", () => { - const match = getAutoSubmitMatch( - "text please submit now", - parseAutoSubmitPhrases("submit now, now"), - ); - assert.ok(match); - assert.equal(match.matchedPhrase, "submit now"); - assert.equal(match.textWithoutPhrase, "text please"); -}); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts index 1c287cef72..c8d66a3ca6 100644 --- a/desktop/src/features/dictation/lib/voiceInput.ts +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -1,40 +1,3 @@ -/** - * Default auto-submit phrase. Empty string disables auto-submit — the user - * must manually press Enter/Send after dictation. The infrastructure for - * configurable phrases is in place (parseAutoSubmitPhrases, getAutoSubmitMatch) - * and can be wired to a user setting when we're ready to ship auto-submit. - */ -export const DEFAULT_AUTO_SUBMIT_PHRASE = ""; - -const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u; - -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function normalizePhrase(value: string): string { - return value - .toLowerCase() - .replace(/\s+/g, " ") - .trim() - .replace(TRAILING_PUNCTUATION_REGEX, "") - .trim(); -} - -export function parseAutoSubmitPhrases( - rawValue: string | null | undefined, -): string[] { - if (!rawValue) return []; - return Array.from( - new Set( - rawValue - .split(",") - .map((value) => normalizePhrase(value)) - .filter(Boolean), - ), - ); -} - function appendTranscribedText(baseText: string, fragment: string): string { const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); if (!normalizedFragment) return baseText; @@ -45,6 +8,22 @@ function appendTranscribedText(baseText: string, fragment: string): string { return `${baseText} ${normalizedFragment}`; } +export type DictationSendDecision = "send" | "stop-recording" | "wait"; + +export function getDictationSendDecision({ + isRecording, + isStarting, + isTranscribing, +}: { + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; +}): DictationSendDecision { + if (isRecording || isStarting) return "stop-recording"; + if (isTranscribing) return "wait"; + return "send"; +} + export function replaceTrailingTranscribedText( fullText: string, previousTranscribedText: string, @@ -71,44 +50,3 @@ export function replaceTrailingTranscribedText( return appendTranscribedText(fullText, nextTranscribedText); } - -export function getAutoSubmitMatch( - transcribedText: string, - autoSubmitPhrases: string[], -): { matchedPhrase: string; textWithoutPhrase: string } | null { - const normalizedTranscribedText = normalizePhrase(transcribedText); - if (!normalizedTranscribedText) return null; - - const sortedPhrases = [...autoSubmitPhrases].sort( - (left, right) => right.length - left.length, - ); - - for (const phrase of sortedPhrases) { - if (!normalizedTranscribedText.endsWith(phrase)) continue; - - const phraseStartIndex = normalizedTranscribedText.length - phrase.length; - if ( - phraseStartIndex > 0 && - normalizedTranscribedText[phraseStartIndex - 1] !== " " - ) { - continue; - } - - const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, ""); - const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp); - const phrasePattern = new RegExp( - `(^|\\s)(${phraseWords.join("\\s+")})\\s*$`, - "iu", - ); - const rawMatch = trimmedText.match(phrasePattern); - const phraseStartOffset = - rawMatch && rawMatch.index !== undefined - ? rawMatch.index + (rawMatch[1]?.length ?? 0) - : trimmedText.length - phrase.length; - const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd(); - - return { matchedPhrase: phrase, textWithoutPhrase }; - } - - return null; -} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index ef040c68dd..ffcb35681e 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -210,7 +210,7 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); - const stopDictationRef = React.useRef<() => void>(() => {}); + const prepareDictationSubmitRef = React.useRef<() => boolean>(() => true); const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler @@ -275,11 +275,10 @@ function MessageComposerImpl({ isUploadingRef, setComposerContent, setEditorContent: richText.setContent, - submitMessageRef, draftKey: effectiveDraftKey, composerRef: composerScrollRef, }); - stopDictationRef.current = dictation.cancelRecording; + prepareDictationSubmitRef.current = dictation.prepareToSubmit; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); @@ -518,12 +517,13 @@ function MessageComposerImpl({ // ── Submit message ────────────────────────────────────────────────── const submitMessage = React.useCallback(async () => { + if (!prepareDictationSubmitRef.current()) return; + const trimmed = syncComposerContentFromEditor().trim(); // Edit mode if (editTargetRef.current && onEditSaveRef.current) { if (isSendingRef.current || isUploadingRef.current) return; - stopDictationRef.current(); const currentPendingImeta = media.pendingImetaRef.current; // No empty-edit guard here: clearing an edit to empty (no text, no // attachments) flows through to onEditSave as empty content, which @@ -599,8 +599,6 @@ function MessageComposerImpl({ ) { return; } - stopDictationRef.current(); - const capturedThreadContext = onCaptureSendContext?.() ?? null; if ( capturedThreadContext !== null && @@ -840,11 +838,13 @@ function MessageComposerImpl({ disabled || media.isUploading || mentionSendFlow.isPreparingMentionSend || + dictation.isSendBlocked || (isContentEmpty && media.pendingImeta.length === 0), [ disabled, media.isUploading, mentionSendFlow.isPreparingMentionSend, + dictation.isSendBlocked, isContentEmpty, media.pendingImeta.length, ], From bb0f1fa95a9ad0bc0ef4d41c9de992e3f2f1ddf6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 11:50:13 -0400 Subject: [PATCH 04/13] fix(desktop): keep dictation send controls active Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- .../features/dictation/hooks/useMessageComposerDictation.tsx | 1 - desktop/src/features/dictation/index.ts | 1 - desktop/src/features/messages/ui/MessageComposer.tsx | 2 -- 3 files changed, 4 deletions(-) diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx index 407d6d9a1e..97252bf485 100644 --- a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -44,7 +44,6 @@ export function useMessageComposerDictation({ return { ...dictation, - isSendBlocked: isRecording || isStarting || isTranscribing, prepareToSubmit, }; } diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts index 502b37e6b9..12964f17af 100644 --- a/desktop/src/features/dictation/index.ts +++ b/desktop/src/features/dictation/index.ts @@ -6,4 +6,3 @@ export { } from "./hooks/useMessageComposerDictation"; export { useVoiceDictationShortcut } from "./hooks/useVoiceDictationShortcut"; export { DictationButton } from "./ui/DictationButton"; -export { getDictationSendDecision } from "./lib/voiceInput"; diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index ffcb35681e..ecee420ab9 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -838,13 +838,11 @@ function MessageComposerImpl({ disabled || media.isUploading || mentionSendFlow.isPreparingMentionSend || - dictation.isSendBlocked || (isContentEmpty && media.pendingImeta.length === 0), [ disabled, media.isUploading, mentionSendFlow.isPreparingMentionSend, - dictation.isSendBlocked, isContentEmpty, media.pendingImeta.length, ], From 3057890dd655690f9c53d4362e3535745fd339bb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 11:57:59 -0400 Subject: [PATCH 05/13] fix(desktop): stop empty dictation from send Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- .../dictation/hooks/useMessageComposerDictation.tsx | 1 + desktop/src/features/messages/ui/MessageComposer.tsx | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx index 97252bf485..a6d5d42774 100644 --- a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -44,6 +44,7 @@ export function useMessageComposerDictation({ return { ...dictation, + canStopFromSend: isRecording || isStarting, prepareToSubmit, }; } diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index ecee420ab9..1fbf8440df 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -518,7 +518,6 @@ function MessageComposerImpl({ // ── Submit message ────────────────────────────────────────────────── const submitMessage = React.useCallback(async () => { if (!prepareDictationSubmitRef.current()) return; - const trimmed = syncComposerContentFromEditor().trim(); // Edit mode @@ -838,13 +837,16 @@ function MessageComposerImpl({ disabled || media.isUploading || mentionSendFlow.isPreparingMentionSend || - (isContentEmpty && media.pendingImeta.length === 0), + (isContentEmpty && + media.pendingImeta.length === 0 && + !dictation.canStopFromSend), [ disabled, media.isUploading, mentionSendFlow.isPreparingMentionSend, isContentEmpty, media.pendingImeta.length, + dictation.canStopFromSend, ], ); From 0da28eff4e1d39c4477858486ad4e2b626292f06 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 12:54:38 -0400 Subject: [PATCH 06/13] fix(desktop): send finalized dictation in one click Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- .../hooks/useMessageComposerDictation.tsx | 36 +++++++++++++++++-- .../dictation/lib/voiceInput.test.mjs | 27 ++++++++++++++ .../src/features/dictation/lib/voiceInput.ts | 14 ++++++++ .../features/messages/ui/MessageComposer.tsx | 2 +- 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx index a6d5d42774..960c390e6b 100644 --- a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -1,7 +1,10 @@ import type * as React from "react"; -import { useCallback, useRef } from "react"; +import { useCallback, useEffect, useRef } from "react"; import { useFeatureEnabled } from "@/shared/features"; -import { getDictationSendDecision } from "../lib/voiceInput"; +import { + getDictationSendDecision, + shouldAutoSubmitDictation, +} from "../lib/voiceInput"; import { DictationButton } from "../ui/DictationButton"; import { useComposerDictation } from "./useComposerDictation"; @@ -15,10 +18,14 @@ interface UseMessageComposerDictationOptions { setEditorContent: (text: string) => void; draftKey: string | null; composerRef: React.RefObject; + submitMessageRef: React.MutableRefObject<() => void>; } export function useMessageComposerDictation({ + disabled, + draftKey, setEditorContent, + submitMessageRef, ...options }: UseMessageComposerDictationOptions) { const enabled = useFeatureEnabled("voiceDictation"); @@ -26,10 +33,34 @@ export function useMessageComposerDictation({ setEditorContentRef.current = setEditorContent; const dictation = useComposerDictation({ ...options, + disabled, + draftKey, enabled, setEditorContentRef, }); const { isRecording, isStarting, isTranscribing, stopRecording } = dictation; + const sendAfterTranscriptionRef = useRef(false); + + // biome-ignore lint/correctness/useExhaustiveDependencies: composer scope and availability are the reset triggers for a queued send + useEffect(() => { + sendAfterTranscriptionRef.current = false; + }, [disabled, draftKey]); + + useEffect(() => { + if ( + !shouldAutoSubmitDictation({ + requested: sendAfterTranscriptionRef.current, + isRecording, + isStarting, + isTranscribing, + }) + ) { + return; + } + sendAfterTranscriptionRef.current = false; + submitMessageRef.current(); + }, [isRecording, isStarting, isTranscribing, submitMessageRef]); + const prepareToSubmit = useCallback(() => { const decision = getDictationSendDecision({ isRecording, @@ -37,6 +68,7 @@ export function useMessageComposerDictation({ isTranscribing, }); if (decision === "stop-recording") { + sendAfterTranscriptionRef.current = true; stopRecording(); } return decision === "send"; diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs index 908a7bd7f8..3360d9ab0f 100644 --- a/desktop/src/features/dictation/lib/voiceInput.test.mjs +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -4,6 +4,7 @@ import test from "node:test"; import { getDictationSendDecision, replaceTrailingTranscribedText, + shouldAutoSubmitDictation, } from "./voiceInput.ts"; test("dictation send stops capture before submitting", () => { @@ -44,6 +45,32 @@ test("dictation send waits for the final transcript", () => { ); }); +test("dictation auto-submit waits until capture and transcription settle", () => { + const state = { + requested: true, + isRecording: false, + isStarting: false, + isTranscribing: false, + }; + assert.equal(shouldAutoSubmitDictation(state), true); + assert.equal( + shouldAutoSubmitDictation({ ...state, requested: false }), + false, + ); + assert.equal( + shouldAutoSubmitDictation({ ...state, isRecording: true }), + false, + ); + assert.equal( + shouldAutoSubmitDictation({ ...state, isStarting: true }), + false, + ); + assert.equal( + shouldAutoSubmitDictation({ ...state, isTranscribing: true }), + false, + ); +}); + // ── replaceTrailingTranscribedText ────────────────────────────────────────── test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts index c8d66a3ca6..da92bdf7fc 100644 --- a/desktop/src/features/dictation/lib/voiceInput.ts +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -24,6 +24,20 @@ export function getDictationSendDecision({ return "send"; } +export function shouldAutoSubmitDictation({ + requested, + isRecording, + isStarting, + isTranscribing, +}: { + requested: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; +}): boolean { + return requested && !isRecording && !isStarting && !isTranscribing; +} + export function replaceTrailingTranscribedText( fullText: string, previousTranscribedText: string, diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 1fbf8440df..5d92b3c25e 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -232,7 +232,6 @@ function MessageComposerImpl({ (replyTarget ? `Reply to ${replyTarget.author} in #${channelName}` : `Message #${channelName}`)); - const richText = useRichTextEditor({ placeholder: computedPlaceholder, editable: !disabled, @@ -277,6 +276,7 @@ function MessageComposerImpl({ setEditorContent: richText.setContent, draftKey: effectiveDraftKey, composerRef: composerScrollRef, + submitMessageRef, }); prepareDictationSubmitRef.current = dictation.prepareToSubmit; const linkEditor = useLinkEditor(richText); From b83f7120df89e46d8fa31b69a1b870e1d898801c Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 13:12:46 -0400 Subject: [PATCH 07/13] fix(desktop): queue sends during dictation finalization Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- .../features/dictation/hooks/useMessageComposerDictation.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx index 960c390e6b..e026c60c30 100644 --- a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -67,8 +67,10 @@ export function useMessageComposerDictation({ isStarting, isTranscribing, }); - if (decision === "stop-recording") { + if (decision !== "send") { sendAfterTranscriptionRef.current = true; + } + if (decision === "stop-recording") { stopRecording(); } return decision === "send"; From f178bc30e2561f508717f6d6e1eec3516ad11bc3 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 31 Jul 2026 14:48:13 -0400 Subject: [PATCH 08/13] chore(desktop): preserve app shell size budget Signed-off-by: John Tennant --- desktop/src/app/AppShell.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index fac09191bd..022bde5d38 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -104,7 +104,6 @@ const LazySettingsScreen = React.lazy(async () => { const module = await import("@/features/settings/ui/SettingsScreen"); return { default: module.SettingsScreen }; }); - export function AppShell() { useWebviewZoomShortcuts(); useTauriWindowDrag(); @@ -152,7 +151,6 @@ export function AppShell() { selectedChannelId, selectedView, }); - // Settings lives in history so back returns to the previous app entry. const settingsOpen = location.pathname === "/settings"; useVoiceDictationShortcut(settingsOpen); const locationSearchSection = (location.search as { section?: unknown }) From 6ac563159d3003af4a0e90d88dbf04e463bc145d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 31 Jul 2026 14:48:43 -0400 Subject: [PATCH 09/13] refactor(desktop): consolidate composer helpers Signed-off-by: John Tennant --- .../src/features/messages/ui/MessageComposer.tsx | 2 +- .../messages/ui/MessageComposerOverlays.tsx | 13 +++++++++++++ .../messages/ui/useComposerScrollToBottom.ts | 13 ------------- 3 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 desktop/src/features/messages/ui/useComposerScrollToBottom.ts diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 5d92b3c25e..c60976de53 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -57,8 +57,8 @@ import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; import { MessageComposerAutocompletes, MessageComposerUploadError, + useComposerScrollToBottom, } from "./MessageComposerOverlays"; -import { useComposerScrollToBottom } from "./useComposerScrollToBottom"; import type { MessageComposerProps } from "./MessageComposer.types"; diff --git a/desktop/src/features/messages/ui/MessageComposerOverlays.tsx b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx index 67b4f030b3..cf2b5fde4a 100644 --- a/desktop/src/features/messages/ui/MessageComposerOverlays.tsx +++ b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx @@ -1,3 +1,4 @@ +import * as React from "react"; import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import type { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; import type { EmojiSuggestion } from "@/features/messages/lib/useEmojiAutocomplete"; @@ -10,6 +11,18 @@ import { type MentionSuggestion, } from "./MentionAutocomplete"; +export function useComposerScrollToBottom( + composerScrollRef: React.RefObject, +) { + return React.useCallback(() => { + window.requestAnimationFrame(() => { + const scrollElement = composerScrollRef.current; + if (!scrollElement) return; + scrollElement.scrollTop = scrollElement.scrollHeight; + }); + }, [composerScrollRef]); +} + export function MessageComposerAutocompletes({ applyChannelInsert, applyEmojiInsert, diff --git a/desktop/src/features/messages/ui/useComposerScrollToBottom.ts b/desktop/src/features/messages/ui/useComposerScrollToBottom.ts deleted file mode 100644 index 9d07efb3a5..0000000000 --- a/desktop/src/features/messages/ui/useComposerScrollToBottom.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as React from "react"; - -export function useComposerScrollToBottom( - composerScrollRef: React.RefObject, -) { - return React.useCallback(() => { - window.requestAnimationFrame(() => { - const scrollElement = composerScrollRef.current; - if (!scrollElement) return; - scrollElement.scrollTop = scrollElement.scrollHeight; - }); - }, [composerScrollRef]); -} From 68e4613c67ef37111bb4ee96abec2c0ce4eb2d76 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 31 Jul 2026 14:51:35 -0400 Subject: [PATCH 10/13] fix(desktop): register dictation commands Signed-off-by: John Tennant --- desktop/src-tauri/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 80b23b7b62..59e7e3b897 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -907,6 +907,10 @@ pub fn run() { list_audio_output_devices, set_audio_output_device, get_audio_output_device, + dictation::start_dictation, + dictation::stop_dictation, + dictation::push_dictation_audio, + dictation::get_dictation_status, start_pairing, confirm_pairing_sas, cancel_pairing, From c831ebea2d3a7c71ea2a9e6d63d335f88bc69451 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 31 Jul 2026 14:52:17 -0400 Subject: [PATCH 11/13] docs(desktop): tighten media proxy comment Signed-off-by: John Tennant --- desktop/src-tauri/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 59e7e3b897..e54d7ab3da 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -495,9 +495,7 @@ pub fn run() { }); } - // Start the localhost media streaming proxy. Uses the shared HTTP - // client so VPN tunnelling applies. The port is stored in AppState - // and exposed to the frontend via the `get_media_proxy_port` command. + // Start the localhost media proxy with the shared VPN-aware HTTP client. let proxy_client = state.http_client.clone(); let proxy_handle = app_handle.clone(); tauri::async_runtime::spawn(async move { From 5e85a241f99f19ecc9800c487af669fa74dbfe3f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 31 Jul 2026 15:02:32 -0400 Subject: [PATCH 12/13] fix(desktop): scope dictation to composer context Signed-off-by: John Tennant --- .../features/dictation/hooks/useDictation.ts | 21 ++------- .../dictation/lib/voiceInput.test.mjs | 46 ++++--------------- .../src/features/dictation/lib/voiceInput.ts | 32 ++----------- .../features/messages/ui/MessageComposer.tsx | 2 +- 4 files changed, 16 insertions(+), 85 deletions(-) diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts index 34a8bf2453..a3ca1b75aa 100644 --- a/desktop/src/features/dictation/hooks/useDictation.ts +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -1,5 +1,5 @@ -import { useCallback, useRef } from "react"; -import { replaceTrailingTranscribedText } from "../lib/voiceInput"; +import { useCallback } from "react"; +import { appendTranscribedText } from "../lib/voiceInput"; import { useLocalDictation } from "./useLocalDictation"; interface UseDictationOptions { @@ -16,30 +16,15 @@ export function useDictation({ getText, setText, }: UseDictationOptions) { - const lastTranscriptRef = useRef(""); - const handleTranscript = useCallback( (transcript: string) => { - const previous = lastTranscriptRef.current; - const latest = getText(); - const merged = replaceTrailingTranscribedText( - latest, - previous, - transcript, - ); - setText(merged); - // Each native flush is an independent segment, so the next transcript - // appends instead of replacing this one. - lastTranscriptRef.current = ""; + setText(appendTranscribedText(getText(), transcript)); }, [getText, setText], ); const dictation = useLocalDictation({ disabled, - onRecordingStart: () => { - lastTranscriptRef.current = ""; - }, onTranscriptText: handleTranscript, }); diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs index 3360d9ab0f..2ae61b3387 100644 --- a/desktop/src/features/dictation/lib/voiceInput.test.mjs +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -2,8 +2,8 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + appendTranscribedText, getDictationSendDecision, - replaceTrailingTranscribedText, shouldAutoSubmitDictation, } from "./voiceInput.ts"; @@ -71,46 +71,16 @@ test("dictation auto-submit waits until capture and transcription settle", () => ); }); -// ── replaceTrailingTranscribedText ────────────────────────────────────────── +// ── appendTranscribedText ─────────────────────────────────────────────────── -test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { - assert.equal( - replaceTrailingTranscribedText("Hello", "", "world"), - "Hello world", - ); -}); - -test("replaceTrailingTranscribedText_appendsToEmptyBase", () => { - assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello"); -}); - -test("replaceTrailingTranscribedText_replacesTrailingInterim", () => { - // Interim "hello wor" is refined to "hello world". - assert.equal( - replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"), - "hello world", - ); +test("appendTranscribedText_appendsToExistingText", () => { + assert.equal(appendTranscribedText("Hello", "world"), "Hello world"); }); -test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => { - // User typed "Note: " then dictated; the manual prefix must survive. - assert.equal( - replaceTrailingTranscribedText("Note: hi", "hi", "hi there"), - "Note: hi there", - ); -}); - -test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => { - // If the previous transcript isn't the trailing text anymore, append. - assert.equal( - replaceTrailingTranscribedText("edited text", "old", "new"), - "edited text new", - ); +test("appendTranscribedText_appendsToEmptyBase", () => { + assert.equal(appendTranscribedText("", "hello"), "hello"); }); -test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { - assert.equal( - replaceTrailingTranscribedText("Hello", "", ", world"), - "Hello, world", - ); +test("appendTranscribedText_avoidsSpaceBeforePunctuation", () => { + assert.equal(appendTranscribedText("Hello", ", world"), "Hello, world"); }); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts index da92bdf7fc..7bb10c3a9f 100644 --- a/desktop/src/features/dictation/lib/voiceInput.ts +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -1,4 +1,7 @@ -function appendTranscribedText(baseText: string, fragment: string): string { +export function appendTranscribedText( + baseText: string, + fragment: string, +): string { const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); if (!normalizedFragment) return baseText; if (!baseText.trim()) return normalizedFragment; @@ -37,30 +40,3 @@ export function shouldAutoSubmitDictation({ }): boolean { return requested && !isRecording && !isStarting && !isTranscribing; } - -export function replaceTrailingTranscribedText( - fullText: string, - previousTranscribedText: string, - nextTranscribedText: string, -): string { - if (!previousTranscribedText) { - return appendTranscribedText(fullText, nextTranscribedText); - } - - if (fullText.endsWith(previousTranscribedText)) { - return appendTranscribedText( - fullText.slice(0, -previousTranscribedText.length), - nextTranscribedText, - ); - } - - const trimmedPreviousText = previousTranscribedText.trim(); - if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) { - return appendTranscribedText( - fullText.slice(0, -trimmedPreviousText.length), - nextTranscribedText, - ); - } - - return appendTranscribedText(fullText, nextTranscribedText); -} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index c60976de53..ce9b6f8ee5 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -274,7 +274,7 @@ function MessageComposerImpl({ isUploadingRef, setComposerContent, setEditorContent: richText.setContent, - draftKey: effectiveDraftKey, + draftKey: `${effectiveDraftKey}\0${editTarget?.id ?? ""}\0${replyTarget?.id ?? ""}`, composerRef: composerScrollRef, submitMessageRef, }); From e43ef004ebd4a42c36e53f9b080dce6ae4326b0d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 31 Jul 2026 15:06:52 -0400 Subject: [PATCH 13/13] refactor(desktop): remove unused dictation start callback Signed-off-by: John Tennant --- desktop/src/features/dictation/hooks/useLocalDictation.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/desktop/src/features/dictation/hooks/useLocalDictation.ts b/desktop/src/features/dictation/hooks/useLocalDictation.ts index 2a4e3ba441..13b40e3592 100644 --- a/desktop/src/features/dictation/hooks/useLocalDictation.ts +++ b/desktop/src/features/dictation/hooks/useLocalDictation.ts @@ -18,7 +18,6 @@ function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { interface UseLocalDictationOptions { disabled?: boolean; - onRecordingStart?: () => void; onTranscriptText: (text: string) => void; } @@ -71,7 +70,6 @@ const SESSION_HEADER_BYTES = 8; */ export function useLocalDictation({ disabled = false, - onRecordingStart, onTranscriptText, }: UseLocalDictationOptions) { const [isRecording, setIsRecording] = useState(false); @@ -91,7 +89,6 @@ export function useLocalDictation({ const flushChainRef = useRef>(Promise.resolve()); const unlistenTranscriptRef = useRef(null); const unlistenStateRef = useRef(null); - const onRecordingStartRef = useRef(onRecordingStart); const onTranscriptTextRef = useRef(onTranscriptText); // Native session ID — set after `start_dictation` returns. Transcript and // state events include this ID so we can definitively ignore stale events @@ -101,7 +98,6 @@ export function useLocalDictation({ // awaiting async setup. The start resumes and bails before activating. const startAbortedRef = useRef(false); - onRecordingStartRef.current = onRecordingStart; onTranscriptTextRef.current = onTranscriptText; const isEnabled = !disabled && isAvailable; @@ -307,7 +303,6 @@ export function useLocalDictation({ flushChainRef.current = Promise.resolve(); setIsStarting(true); - onRecordingStartRef.current?.(); try { // 1. Start the native STT engine — returns the session ID used to tag events.