diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000000..dc1aed2279 --- /dev/null +++ b/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,252 @@ +//! Local dictation pipeline — uses the Parakeet STT engine for offline +//! speech-to-text in the message composer. +//! +//! Unlike the huddle STT pipeline (which posts kind:9 events to the relay), +//! dictation emits transcribed text back to the frontend via Tauri events +//! so the composer can display it in real-time. +//! +//! Key differences from huddle STT: +//! - No TTS barge-in / echo gating (no agent voice in composer context) +//! - No shared huddle PTT gate; the composer controls its own session +//! - Slightly longer silence threshold for more coherent sentences +//! - Text goes to the frontend, not to the relay + +use std::sync::{Arc, LazyLock, Mutex}; + +use tauri::{Emitter, State}; + +use crate::app_state::AppState; +use crate::huddle::{models, stt::SttPipeline}; + +/// Tauri event name emitted when a dictation transcript segment is ready. +const DICTATION_TRANSCRIPT_EVENT: &str = "dictation-transcript"; + +/// Tauri event name emitted when dictation state changes (started/stopped). +const DICTATION_STATE_EVENT: &str = "dictation-state"; + +/// State for the active dictation session. +/// +/// Stored app-wide behind a `Mutex`. Only one dictation session can be active +/// at a time (starting a new one stops the previous). +pub(crate) struct DictationState { + /// The running STT engine, if dictation is active. + engine: Option>, + /// Monotonically increasing session counter. Included in all emitted events + /// so the frontend can ignore stale transcripts from a previous session's + /// forwarder that arrive after a new session has started. + session_id: u64, +} + +impl DictationState { + pub fn new() -> Self { + Self { + engine: None, + session_id: 0, + } + } +} + +static DICTATION_STATE: LazyLock> = + LazyLock::new(|| Mutex::new(DictationState::new())); + +/// `start_dictation` — begin local STT dictation. +/// +/// Starts the Parakeet STT engine and spawns a task that emits +/// `dictation-transcript` events to the frontend as text is recognized. +/// Returns an error if models are not downloaded yet. +#[tauri::command] +pub async fn start_dictation(state: State<'_, AppState>) -> Result { + // Check if models are ready. + if !models::is_stt_ready() { + // Kick off download if not already in progress. + if let Some(mgr) = models::global_model_manager() { + mgr.start_stt_download(state.http_client.clone()); + } + return Err("STT model not ready — download in progress".to_string()); + } + + let model_dir = models::stt_model_dir().ok_or("STT model directory not found")?; + + // Stop any existing dictation session first. + stop_dictation_inner(None); + + let (engine, text_rx) = SttPipeline::new_dictation(model_dir)?; + let engine = Arc::new(engine); + + // Store the engine in state and increment the session counter. + let session_id = { + let mut ds = DICTATION_STATE.lock().unwrap_or_else(|e| e.into_inner()); + ds.engine = Some(Arc::clone(&engine)); + ds.session_id += 1; + ds.session_id + }; + + // Spawn a task that forwards transcribed text to the frontend. + let app_handle = state + .app_handle + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + + if let Some(handle) = app_handle { + let _ = handle.emit( + DICTATION_STATE_EVENT, + serde_json::json!({ "state": "started", "session": session_id }), + ); + spawn_dictation_forwarder(text_rx, handle, session_id); + } + + Ok(session_id) +} + +/// `stop_dictation` — stop the active dictation session. +/// +/// The final transcript (if any) is emitted asynchronously by the forwarder +/// task. The `dictation-state: stopped` event is emitted by the forwarder +/// after all pending transcripts have been forwarded, ensuring the frontend +/// receives the final text before the stopped signal. +/// +/// `session` scopes the stop to a specific session: the engine is only torn +/// down when the currently-stored `session_id` matches. This prevents a +/// delayed/fire-and-forget stop from an old session (e.g. one deferred behind +/// a final audio flush) from killing a *newer* session the user started in the +/// meantime. Pass `None` for an unconditional stop (used on cancel/unmount). +#[tauri::command] +pub fn stop_dictation(session: Option) -> Result<(), String> { + stop_dictation_inner(session); + // Note: `stopped` is emitted by the forwarder task after draining all + // pending transcripts — not here. This avoids a race where the frontend + // sees `stopped` before the final transcript arrives. + Ok(()) +} + +/// `push_dictation_audio` — feed raw PCM bytes into the dictation pipeline. +/// +/// Expects a raw binary body: an 8-byte little-endian `u64` session header +/// followed by f32 LE samples at 48 kHz mono. The header scopes the push to a +/// specific session — bytes are only fed to the engine when the header matches +/// the currently-stored `session_id`. This prevents late audio from a +/// just-stopped session (whose final `flushAudioBatch()` chunks are still +/// arriving) from being accepted by a *newer* session the user started in the +/// meantime and transcribed into the new draft. +/// +/// If no dictation session is active, or the session header doesn't match the +/// active session, the bytes are silently discarded. +#[tauri::command] +pub fn push_dictation_audio(request: tauri::ipc::Request<'_>) -> Result<(), String> { + /// Size of the leading little-endian `u64` session header. + const SESSION_HEADER_BYTES: usize = 8; + /// Maximum IPC audio batch size (audio payload only, excluding the header): 100 KB. + const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() < SESSION_HEADER_BYTES { + return Err(format!( + "audio batch too small: {} bytes (need at least {} for session header)", + bytes.len(), + SESSION_HEADER_BYTES + )); + } + let (header, audio) = bytes.split_at(SESSION_HEADER_BYTES); + if audio.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + audio.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + // `split_at` guarantees `header` is exactly `SESSION_HEADER_BYTES` long. + let session = u64::from_le_bytes( + header + .try_into() + .map_err(|_| "invalid session header".to_string())?, + ); + let ds = DICTATION_STATE.lock().unwrap_or_else(|e| e.into_inner()); + // Only feed audio tagged with the currently-active session. Late + // chunks from an old session are silently dropped. + if session == ds.session_id { + if let Some(ref engine) = ds.engine { + engine.push_audio(audio.to_vec())?; + } + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} + +/// `get_dictation_status` — check if local dictation is available and/or active. +#[tauri::command] +pub fn get_dictation_status() -> DictationStatus { + let model_ready = models::is_stt_ready(); + let is_active = DICTATION_STATE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .engine + .is_some(); + + DictationStatus { + available: model_ready, + active: is_active, + } +} + +/// Response for `get_dictation_status`. +#[derive(serde::Serialize, Clone)] +pub struct DictationStatus { + /// Whether the local STT model is downloaded and ready. + pub available: bool, + /// Whether a dictation session is currently active. + pub active: bool, +} + +// ── Internal helpers ────────────────────────────────────────────────────────── + +fn stop_dictation_inner(session: Option) { + let old_engine = { + let mut ds = DICTATION_STATE.lock().unwrap_or_else(|e| e.into_inner()); + // Session-scoped stop: only tear down when the requested session matches + // the one currently stored. A `None` session stops unconditionally. + match session { + Some(requested) if requested != ds.session_id => None, + _ => ds.engine.take(), + } + }; + if let Some(engine) = old_engine { + engine.shutdown(); + // Drop outside the lock — thread join may block briefly. + drop(engine); + } +} + +/// Spawn an async task that reads transcribed text and emits Tauri events. +/// +/// Each event includes the `session` ID so the frontend can ignore stale +/// transcripts from a previous session's forwarder. When the channel closes +/// (engine stopped), the forwarder emits `dictation-state: stopped`. +fn spawn_dictation_forwarder( + mut text_rx: tokio::sync::mpsc::Receiver, + app_handle: tauri::AppHandle, + session_id: u64, +) { + tauri::async_runtime::spawn(async move { + while let Some(text) = text_rx.recv().await { + if text.is_empty() { + continue; + } + let payload = serde_json::json!({ "text": text, "session": session_id }); + if app_handle + .emit(DICTATION_TRANSCRIPT_EVENT, payload) + .is_err() + { + break; // App window closed. + } + } + // All transcripts forwarded — signal the frontend that dictation is done. + let _ = app_handle.emit( + DICTATION_STATE_EVENT, + serde_json::json!({ "state": "stopped", "session": session_id }), + ); + }); +} diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 6f502ca72c..a66c63ae75 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -41,6 +41,23 @@ const AUDIO_QUEUE_DEPTH: usize = 50; /// Prevents OOM if VAD stays in speech mode (noisy environment). const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; +/// Dictation waits slightly longer for a natural pause before flushing. +const DICTATION_SILENCE_FLUSH_FRAMES: usize = 25; + +/// Emit a partial dictation result after two seconds of uninterrupted speech. +const DICTATION_PARTIAL_FLUSH_SAMPLES: usize = 16_000 * 2; + +struct SttPipelineConfig { + model_dir: PathBuf, + silence_flush_frames: usize, + max_speech_samples: usize, + partial_flush_samples: Option, + tts_active: Arc, + tts_cancel: Option>, + ptt_active: Option>, + flush_on_shutdown: bool, +} + /// Handle to the running STT pipeline. /// /// Not Clone — wrap in `Arc` to share across threads. @@ -88,27 +105,46 @@ impl SttPipeline { tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + Self::new_with_config(SttPipelineConfig { + model_dir, + silence_flush_frames: SILENCE_FLUSH_FRAMES, + max_speech_samples: MAX_SPEECH_SAMPLES, + partial_flush_samples: None, + tts_active, + tts_cancel, + ptt_active, + flush_on_shutdown: false, + }) + } + + /// Spawn an offline STT pipeline for message-composer dictation. + pub(crate) fn new_dictation( + model_dir: PathBuf, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + Self::new_with_config(SttPipelineConfig { + model_dir, + silence_flush_frames: DICTATION_SILENCE_FLUSH_FRAMES, + max_speech_samples: MAX_SPEECH_SAMPLES, + partial_flush_samples: Some(DICTATION_PARTIAL_FLUSH_SAMPLES), + tts_active: Arc::new(AtomicBool::new(false)), + tts_cancel: None, + ptt_active: None, + flush_on_shutdown: true, + }) + } + + fn new_with_config( + config: SttPipelineConfig, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); let shutdown = Arc::new(AtomicBool::new(false)); let shutdown_worker = Arc::clone(&shutdown); - let tts_cancel_worker = tts_cancel.as_ref().map(Arc::clone); - let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); let handle = thread::Builder::new() .name("stt-worker".into()) - .spawn(move || { - stt_worker( - model_dir, - audio_rx, - text_tx, - shutdown_worker, - tts_active, - tts_cancel_worker, - ptt_active_worker, - ) - }) + .spawn(move || stt_worker(config, audio_rx, text_tx, shutdown_worker)) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; let pipeline = Self { @@ -184,6 +220,34 @@ const VAD_THRESHOLD: f32 = 0.5; /// How long the worker waits on the audio channel before checking the shutdown flag. const RECV_TIMEOUT: Duration = Duration::from_millis(50); +enum AudioReceive { + Bytes(Vec), + Retry, + Stop, +} + +fn receive_audio( + audio_rx: &Receiver>, + is_shutting_down: bool, + flush_on_shutdown: bool, +) -> AudioReceive { + if is_shutting_down { + if !flush_on_shutdown { + return AudioReceive::Stop; + } + return match audio_rx.try_recv() { + Ok(bytes) => AudioReceive::Bytes(bytes), + Err(mpsc::TryRecvError::Empty | mpsc::TryRecvError::Disconnected) => AudioReceive::Stop, + }; + } + + match audio_rx.recv_timeout(RECV_TIMEOUT) { + Ok(bytes) => AudioReceive::Bytes(bytes), + Err(mpsc::RecvTimeoutError::Timeout) => AudioReceive::Retry, + Err(mpsc::RecvTimeoutError::Disconnected) => AudioReceive::Stop, + } +} + /// 50 ms cooldown after TTS stops before STT re-enables. /// Prevents the tail of TTS audio from being transcribed as speech. /// Previous value (200 ms) was eating the first word when the user spoke @@ -202,13 +266,10 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(50); const STT_NUM_THREADS: i32 = 1; fn stt_worker( - model_dir: PathBuf, + config: SttPipelineConfig, audio_rx: Receiver>, text_tx: tokio_mpsc::Sender, shutdown: Arc, - tts_active: Arc, - tts_cancel: Option>, - ptt_active: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -235,12 +296,12 @@ fn stt_worker( // in k2-fsa/sherpa-onnx.) use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; - let tokens_path = model_dir.join("tokens.txt"); - let model_path = model_dir.join("model.int8.onnx"); + let tokens_path = config.model_dir.join("tokens.txt"); + let model_path = config.model_dir.join("model.int8.onnx"); if !tokens_path.exists() || !model_path.exists() { eprintln!( "buzz-desktop: STT model not found at {} — STT disabled", - model_dir.display() + config.model_dir.display() ); drain_until_shutdown(audio_rx, &shutdown); return; @@ -281,17 +342,15 @@ fn stt_worker( // ── 5. Main loop ────────────────────────────────────────────────────────── let mut tts_was_active = false; - let mut ptt_was_active = ptt_active + let mut ptt_was_active = config + .ptt_active .as_ref() .is_some_and(|p| p.load(Ordering::Acquire)); loop { - // Check shutdown flag before blocking. - if shutdown.load(Ordering::Acquire) { - break; - } + let is_shutting_down = shutdown.load(Ordering::Acquire); // Track TTS transitions to set the cooldown timer. - let tts_now = tts_active.load(Ordering::Acquire); + let tts_now = config.tts_active.load(Ordering::Acquire); if tts_was_active && !tts_now { // TTS just stopped — record the timestamp for the cooldown window. tts_stopped_at = Some(std::time::Instant::now()); @@ -302,7 +361,7 @@ fn stt_worker( // The worklet stops sending frames when PTT is inactive, so the normal // silence-accumulation flush path never runs. We must flush here on the // active→inactive edge to avoid buffering speech across PTT presses. - if let Some(ref ptt) = ptt_active { + if let Some(ref ptt) = config.ptt_active { let ptt_now = ptt.load(Ordering::Acquire); if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { flush_to_stt(&speech_buf, &recognizer, &text_tx); @@ -313,11 +372,13 @@ fn stt_worker( ptt_was_active = ptt_now; } - // Use recv_timeout so we can periodically check the shutdown flag. - let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { - Ok(b) => b, - Err(mpsc::RecvTimeoutError::Timeout) => continue, - Err(mpsc::RecvTimeoutError::Disconnected) => break, // Sender dropped. + // Dictation drains audio already accepted by the IPC command before + // flushing its final transcript. Huddles retain their immediate + // shutdown behavior so leaving cannot publish a late transcript. + let bytes = match receive_audio(&audio_rx, is_shutting_down, config.flush_on_shutdown) { + AudioReceive::Bytes(bytes) => bytes, + AudioReceive::Retry => continue, + AudioReceive::Stop => break, }; // Drain any additional pending messages to batch-process. @@ -345,18 +406,45 @@ fn stt_worker( &mut barge_in_frames, &recognizer, &text_tx, - &tts_active, - tts_cancel.as_deref(), + &config.tts_active, + config.tts_cancel.as_deref(), &mut tts_stopped_at, - ptt_active.as_ref(), + config.ptt_active.as_ref(), + config.silence_flush_frames, + config.max_speech_samples, + config.partial_flush_samples, ); } } } - // No final flush — leave_huddle/end_huddle emit lifecycle events before - // the STT worker exits, so a final flush would post a kind:9 message AFTER - // the user has "left." Losing the last partial utterance is acceptable. + if config.flush_on_shutdown { + if !input_buf_48k.is_empty() { + input_buf_48k.resize(chunk_in, 0.0); + let resampled = resample_chunk(&mut resampler, &input_buf_48k); + process_16k_samples( + &resampled, + &mut leftover_16k, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + &config.tts_active, + config.tts_cancel.as_deref(), + &mut tts_stopped_at, + config.ptt_active.as_ref(), + config.silence_flush_frames, + config.max_speech_samples, + config.partial_flush_samples, + ); + } + if !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + } + } } /// Resample a mono 48 kHz chunk to 16 kHz using rubato. @@ -411,6 +499,9 @@ fn process_16k_samples( tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, ptt_active: Option<&Arc>, + silence_flush_frames: usize, + max_speech_samples: usize, + partial_flush_samples: Option, ) { leftover.extend_from_slice(samples); @@ -494,7 +585,9 @@ fn process_16k_samples( speech_buf.extend_from_slice(&frame); // OOM guard: flush and reset if the buffer exceeds 30 s of audio. - if speech_buf.len() >= MAX_SPEECH_SAMPLES { + if speech_buf.len() >= max_speech_samples + || partial_flush_samples.is_some_and(|limit| speech_buf.len() >= limit) + { flush_to_stt(speech_buf, recognizer, text_tx); speech_buf.clear(); *silence_frames = 0; @@ -509,7 +602,7 @@ fn process_16k_samples( // key-hold as one utterance. The PTT release edge in the main // loop handles the flush. In VAD mode, flush after the silence // threshold so each natural pause becomes a separate message. - if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { + if ptt_active.is_none() && *silence_frames >= silence_flush_frames { // End of utterance — transcribe. flush_to_stt(speech_buf, recognizer, text_tx); speech_buf.clear(); @@ -565,3 +658,36 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. use super::drain_until_shutdown; + +#[cfg(test)] +mod tests { + use std::sync::mpsc; + + use super::{receive_audio, AudioReceive}; + + #[test] + fn dictation_shutdown_drains_accepted_audio() { + let (audio_tx, audio_rx) = mpsc::sync_channel(1); + audio_tx.send(vec![1, 2, 3, 4]).unwrap(); + + assert!(matches!( + receive_audio(&audio_rx, true, true), + AudioReceive::Bytes(bytes) if bytes == [1, 2, 3, 4] + )); + assert!(matches!( + receive_audio(&audio_rx, true, true), + AudioReceive::Stop + )); + } + + #[test] + fn huddle_shutdown_does_not_drain_accepted_audio() { + let (audio_tx, audio_rx) = mpsc::sync_channel(1); + audio_tx.send(vec![1, 2, 3, 4]).unwrap(); + + assert!(matches!( + receive_audio(&audio_rx, true, false), + AudioReceive::Stop + )); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 6814008f0d..e54d7ab3da 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -4,6 +4,7 @@ mod archive; mod builderlab; mod commands; mod deep_link; +mod dictation; mod egress_guard; mod event_sync; mod events; @@ -494,9 +495,7 @@ pub fn run() { }); } - // Start the localhost media streaming proxy. Uses the shared HTTP - // client so VPN tunnelling applies. The port is stored in AppState - // and exposed to the frontend via the `get_media_proxy_port` command. + // Start the localhost media proxy with the shared VPN-aware HTTP client. let proxy_client = state.http_client.clone(); let proxy_handle = app_handle.clone(); tauri::async_runtime::spawn(async move { @@ -906,6 +905,10 @@ pub fn run() { list_audio_output_devices, set_audio_output_device, get_audio_output_device, + dictation::start_dictation, + dictation::stop_dictation, + dictation::push_dictation_audio, + dictation::get_dictation_status, start_pairing, confirm_pairing_sas, cancel_pairing, diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 4eb0a42bbe..022bde5d38 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -86,6 +86,7 @@ import { useIdentityQuery } from "@/shared/api/hooks"; import { useRelayAutoHeal } from "@/shared/api/useRelayAutoHeal"; import { useDeferredStartup } from "@/shared/hooks/useDeferredStartup"; import { useWebviewScrollBoundaryLock } from "@/shared/hooks/useWebviewScrollBoundaryLock"; +import { useVoiceDictationShortcut } from "@/features/dictation"; import { joinChannel } from "@/shared/api/tauri"; import type { ChannelVisibility, SearchHit } from "@/shared/api/types"; import { ChannelNavigationProvider } from "@/shared/context/ChannelNavigationContext"; @@ -103,7 +104,6 @@ const LazySettingsScreen = React.lazy(async () => { const module = await import("@/features/settings/ui/SettingsScreen"); return { default: module.SettingsScreen }; }); - export function AppShell() { useWebviewZoomShortcuts(); useTauriWindowDrag(); @@ -151,8 +151,8 @@ export function AppShell() { selectedChannelId, selectedView, }); - // Settings lives in history so back returns to the previous app entry. const settingsOpen = location.pathname === "/settings"; + useVoiceDictationShortcut(settingsOpen); const locationSearchSection = (location.search as { section?: unknown }) .section; const settingsSection: SettingsSection = isSettingsSection( diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts new file mode 100644 index 0000000000..b8869f8033 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -0,0 +1,190 @@ +import type * as React from "react"; +import { useEffect, useId, useRef } from "react"; +import { + clearActiveDictationComposer, + isActiveDictationComposer, + setActiveDictationComposer, +} from "../lib/activeComposer"; +import { useDictation } from "./useDictation"; + +interface UseComposerDictationOptions { + /** Whether the voice-dictation preview feature is enabled. */ + enabled?: boolean; + /** Ref to a function that syncs contentRef from the Tiptap editor and returns it. */ + syncContentRef: React.MutableRefObject<() => string>; + /** Whether the composer is currently disabled (read-only, etc.). */ + disabled?: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + /** Updates contentRef + isContentEmpty state. */ + setComposerContent: (text: string) => void; + /** Ref to a function that updates the Tiptap editor document. */ + setEditorContentRef: React.MutableRefObject<(text: string) => void>; + /** When this key changes (channel/thread switch), active dictation is stopped. */ + draftKey?: string | null; + /** Ref to the composer's container element for focus tracking. */ + composerRef?: React.RefObject; +} + +/** + * Thin wrapper around `useDictation` pre-wired for the MessageComposer's + * state management (syncContentRef, setComposerContent, editor). + * + * Uses the local Parakeet STT engine — fully offline, no relay or API key needed. + */ +export function useComposerDictation({ + enabled = true, + syncContentRef, + disabled = false, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + draftKey, + composerRef, +}: UseComposerDictationOptions) { + const instanceId = useId(); + const isSendBlockedRef = useRef(false); + isSendBlockedRef.current = + !enabled || + disabledRef.current || + isSendingRef.current || + isUploadingRef.current; + + const dictation = useDictation({ + disabled: !enabled, + getText: () => syncContentRef.current(), + setText: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + }, + }); + const startRecordingRef = useRef(dictation.startRecording); + const stopRecordingRef = useRef(dictation.stopRecording); + const isRecordingRef = useRef(dictation.isRecording); + const isStartingRef = useRef(dictation.isStarting); + const shortcutHeldRef = useRef(false); + startRecordingRef.current = dictation.startRecording; + stopRecordingRef.current = dictation.stopRecording; + isRecordingRef.current = dictation.isRecording; + isStartingRef.current = dictation.isStarting; + + // Track which composer is active (most recently focused) so that the global + // ⌘D shortcut only dispatches to one instance when multiple are mounted. + useEffect(() => { + const el = composerRef?.current; + if (!el) { + // If no ref provided, this composer is always considered active (single-composer case). + setActiveDictationComposer(instanceId); + return () => clearActiveDictationComposer(instanceId); + } + + function handleFocusIn() { + setActiveDictationComposer(instanceId); + } + + el.addEventListener("focusin", handleFocusIn); + return () => { + el.removeEventListener("focusin", handleFocusIn); + clearActiveDictationComposer(instanceId); + }; + }, [instanceId, composerRef]); + + // Cancel dictation when the channel/thread changes so that transcript events + // from a stale local STT session don't leak into the wrong draft. + // Only cancel if this instance owns a live/pending session — avoids killing + // another composer's session since the native engine is a singleton. + // + // Includes `isTranscribing`, not just `isRecording`/`isStarting`: + // `stopRecording()` clears `isRecording` immediately but deliberately keeps + // the transcript listener alive (isTranscribing=true) until the native + // `stopped` event, so the final local STT flush can still be appended. If the + // draftKey changes during that grace window, we must still cancel — otherwise + // the late transcript is appended through this same composer instance into the + // newly restored draft. `cancelRecording()` unlistens the transcript handler + // and performs a session-scoped native stop, so this is safe. + const isOwningSessionRef = useRef(false); + isOwningSessionRef.current = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + // biome-ignore lint/correctness/useExhaustiveDependencies: draftKey is the sole trigger + useEffect(() => { + if (isOwningSessionRef.current) { + dictation.cancelRecording(); + } + }, [draftKey]); + + // Auto-cancel dictation when the composer becomes disabled mid-session + // (e.g. channel becomes read-only, parent send state disables thread composer). + // Without this, the STT session keeps running with no way to stop it. + // + // Covers the full ownership window — `isStarting` and `isTranscribing`, not + // just `isRecording`. If the composer is disabled while `startRecording()` is + // still resolving (mic permission / AudioWorklet setup), the pending start + // would otherwise finish and begin microphone capture in a disabled composer. + // `cancelRecording()` aborts the in-flight start as well as a live recording, + // and re-running when any of these flags flip catches a start that completes + // after the disable. + useEffect(() => { + const owningSession = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + if ((!enabled || disabled) && owningSession) { + dictation.cancelRecording(); + } + }, [ + enabled, + disabled, + dictation.isRecording, + dictation.isStarting, + dictation.isTranscribing, + dictation.cancelRecording, + ]); + + // ⌘D push-to-talk — hold to record, release to stop. + // Dispatched from AppShell's keydown/keyup handlers. + // Only the active (most recently focused) composer responds, and only while + // focus remains inside it and it is not disabled/send-blocked. + // biome-ignore lint/correctness/useExhaustiveDependencies: disabledRef/isSendBlockedRef are stable refs read at call time + useEffect(() => { + function handleKeyDown() { + if (!enabled) return; + // Only respond if this is the active composer instance. + if (!isActiveDictationComposer(instanceId)) return; + // Only respond if focus is still inside this composer. The active-composer + // registration persists after focusout (until unmount), so without this + // check, focusing a composer and then moving to another mounted input or + // dialog (quick search, create-channel, etc.) would still let ⌘D start + // microphone capture and append transcript into the background draft. + const el = composerRef?.current; + if (el && !el.contains(document.activeElement)) return; + // Don't start dictation in disabled/blocked composers. + if (disabledRef.current || isSendBlockedRef.current) return; + if ( + !shortcutHeldRef.current && + !isRecordingRef.current && + !isStartingRef.current + ) { + shortcutHeldRef.current = true; + void startRecordingRef.current(); + } + } + function handleKeyUp() { + if (!shortcutHeldRef.current) return; + shortcutHeldRef.current = false; + stopRecordingRef.current(); + } + window.addEventListener("buzz:dictation-key-down", handleKeyDown); + window.addEventListener("buzz:dictation-key-up", handleKeyUp); + return () => { + window.removeEventListener("buzz:dictation-key-down", handleKeyDown); + window.removeEventListener("buzz:dictation-key-up", handleKeyUp); + if (shortcutHeldRef.current) { + shortcutHeldRef.current = false; + stopRecordingRef.current(); + } + }; + }, [instanceId, composerRef, enabled]); + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts new file mode 100644 index 0000000000..a3ca1b75aa --- /dev/null +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -0,0 +1,32 @@ +import { useCallback } from "react"; +import { appendTranscribedText } from "../lib/voiceInput"; +import { useLocalDictation } from "./useLocalDictation"; + +interface UseDictationOptions { + /** Disable native availability checks and recording entry points. */ + disabled?: boolean; + /** Returns the current composer text (must be fresh — synced from editor). */ + getText: () => string; + /** Set composer text */ + setText: (value: string) => void; +} + +export function useDictation({ + disabled = false, + getText, + setText, +}: UseDictationOptions) { + const handleTranscript = useCallback( + (transcript: string) => { + setText(appendTranscribedText(getText(), transcript)); + }, + [getText, setText], + ); + + const dictation = useLocalDictation({ + disabled, + onTranscriptText: handleTranscript, + }); + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useLocalDictation.ts b/desktop/src/features/dictation/hooks/useLocalDictation.ts new file mode 100644 index 0000000000..13b40e3592 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useLocalDictation.ts @@ -0,0 +1,614 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; + +/** + * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. + * Same pattern as huddle's audioWorklet.ts. + */ +function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { + // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition + const internals = (window as any).__TAURI_INTERNALS__; + if (!internals?.invoke) { + return Promise.reject(new Error("Tauri internals not available")); + } + return internals.invoke(cmd, payload); +} + +interface UseLocalDictationOptions { + disabled?: boolean; + onTranscriptText: (text: string) => void; +} + +interface DictationStatus { + available: boolean; + active: boolean; +} + +const DICTATION_TRANSCRIPT_EVENT = "dictation-transcript"; +const DICTATION_STATE_EVENT = "dictation-state"; + +/** Interval (ms) to poll model availability after initial unavailability. */ +const MODEL_POLL_INTERVAL_MS = 5_000; + +/** + * Batching interval for audio IPC (ms). The worklet posts every render quantum + * (~2.67ms at 48kHz/128 samples). Sending each one individually overloads IPC. + * We accumulate ~100ms of audio before sending to reduce IPC overhead from + * ~375 calls/s to ~10 calls/s. + */ +const AUDIO_BATCH_MS = 100; + +/** + * Max samples per `push_dictation_audio` IPC call. The native command rejects + * any raw audio payload over 100 KB (`MAX_AUDIO_BATCH_BYTES` in `dictation.rs`); + * at 48 kHz f32 mono that is 25,600 samples (~0.53s). We chunk under that cap + * (24,000 samples / 96 KB, leaving headroom) so a stalled main thread that + * lets the batch grow past ~0.5s can't produce a single oversized buffer that + * native rejects and we silently drop. Chunks are sent in order. + */ +const MAX_IPC_SAMPLES = 24_000; + +/** + * Size (bytes) of the little-endian `u64` session header prepended to each + * `push_dictation_audio` payload. Native reads this header and only feeds audio + * whose session matches the currently-active one, so late chunks from a + * just-stopped session can't leak into a newer session's transcript. + */ +const SESSION_HEADER_BYTES = 8; + +/** + * Local STT dictation hook using the Parakeet model via Tauri native commands. + * + * Works fully offline — no relay or OpenAI API key needed. Uses the same + * sherpa-onnx Parakeet TDT-CTC 110M model as huddle transcription. + * + * Audio capture uses the Web Audio API (AudioWorklet) on the frontend side, + * then sends batched raw PCM bytes to the native STT engine via + * `push_dictation_audio`. + */ +export function useLocalDictation({ + disabled = false, + onTranscriptText, +}: UseLocalDictationOptions) { + const [isRecording, setIsRecording] = useState(false); + const [isStarting, setIsStarting] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isAvailable, setIsAvailable] = useState(false); + + const streamRef = useRef(null); + const audioContextRef = useRef(null); + const workletRef = useRef(null); + const outputGainRef = useRef(null); + const batchTimerRef = useRef | null>(null); + const audioBatchRef = useRef([]); + // Tail of the serialized flush chain. Each flush appends its IPC work to + // this promise so flushes run strictly one-after-another; awaiting the tail + // guarantees every already-started flush's chunks are enqueued native-side. + const flushChainRef = useRef>(Promise.resolve()); + const unlistenTranscriptRef = useRef(null); + const unlistenStateRef = useRef(null); + const onTranscriptTextRef = useRef(onTranscriptText); + // Native session ID — set after `start_dictation` returns. Transcript and + // state events include this ID so we can definitively ignore stale events + // from a previous session's forwarder. + const nativeSessionRef = useRef(0); + // Abort flag — set when stop/cancel is called while startRecording is still + // awaiting async setup. The start resumes and bails before activating. + const startAbortedRef = useRef(false); + + onTranscriptTextRef.current = onTranscriptText; + + const isEnabled = !disabled && isAvailable; + + // Check availability on mount and poll until available (model may be downloading). + useEffect(() => { + if (disabled) { + setIsAvailable(false); + return; + } + + let cancelled = false; + let pollTimer: ReturnType | null = null; + + function checkAvailability() { + invoke("get_dictation_status") + .then((status) => { + if (cancelled) return; + setIsAvailable(status.available); + // Stop polling once the model is ready. + if (status.available && pollTimer) { + clearInterval(pollTimer); + pollTimer = null; + } + }) + .catch(() => { + if (!cancelled) setIsAvailable(false); + }); + } + + checkAvailability(); + + // Poll periodically until available (handles background model download). + pollTimer = setInterval(() => { + if (cancelled) return; + checkAvailability(); + }, MODEL_POLL_INTERVAL_MS); + + return () => { + cancelled = true; + if (pollTimer) clearInterval(pollTimer); + }; + }, [disabled]); + + /** Flush accumulated audio batch to the native STT engine. Returns a promise + * that resolves once the IPC call completes (or immediately if nothing to flush). + * + * Flushes are serialized through `flushChainRef`: the batch is drained + * synchronously here (so each flush owns a distinct set of samples in FIFO + * order), but the actual IPC send is chained after any prior in-flight + * flush. This keeps chunks strictly ordered and lets `stopRecording`/ + * `cleanup` await the chain tail so an earlier timer-tick flush can't still + * be enqueuing chunks when `stop_dictation` fires. */ + const flushAudioBatch = useCallback((): Promise => { + const batch = audioBatchRef.current; + if (batch.length === 0) return flushChainRef.current; + + // Tag this flush with the session that owns the buffered audio. Captured + // once here so every chunk of this batch carries the same session; native + // drops chunks whose session no longer matches the active one, so a late + // flush from a just-stopped session can't leak into a newer session. + const session = nativeSessionRef.current; + + // Calculate total byte length and merge into a single buffer. Drain the + // batch synchronously so a concurrent timer tick can't grab the same + // samples — the merged buffer captured here is this flush's exclusive + // payload. + let totalSamples = 0; + for (const chunk of batch) { + totalSamples += chunk.length; + } + const merged = new Float32Array(totalSamples); + let offset = 0; + for (const chunk of batch) { + merged.set(chunk, offset); + offset += chunk.length; + } + audioBatchRef.current = []; + + // Split into chunks under the native IPC cap and send them in order. + // A single unbounded buffer can exceed the 100 KB native limit if the + // batch timer was delayed (e.g. main-thread stall), in which case native + // rejects it and the whole chunk is silently lost. + const sendChunks = async (): Promise => { + for (let start = 0; start < merged.length; start += MAX_IPC_SAMPLES) { + const slice = merged.subarray( + start, + Math.min(start + MAX_IPC_SAMPLES, merged.length), + ); + // Build the IPC payload: an 8-byte LE u64 session header followed by + // the audio bytes. Copy the slice into the header-prefixed buffer so + // the raw payload is exactly this chunk (a subarray view shares the + // parent's ArrayBuffer). + const payload = new Uint8Array(SESSION_HEADER_BYTES + slice.byteLength); + new DataView(payload.buffer).setBigUint64( + 0, + BigInt(session), + true, // little-endian + ); + payload.set( + new Uint8Array( + slice.buffer.slice( + slice.byteOffset, + slice.byteOffset + slice.byteLength, + ), + ), + SESSION_HEADER_BYTES, + ); + await invokeRawBinary("push_dictation_audio", payload).catch(() => {}); + } + }; + + // Chain this flush's IPC work after any prior in-flight flush so chunks + // stay strictly ordered and awaiting the tail drains everything. + const chained = flushChainRef.current.then(sendChunks); + flushChainRef.current = chained; + return chained; + }, []); + + const cleanup = useCallback(() => { + // Abort any in-flight startRecording so it bails after its next await + // instead of resuming and opening the mic/worklet or leaving the native + // session/listeners running after teardown. `cleanup` is the unmount + // handler (and the catch-path teardown); without this, unmounting while + // startRecording awaits `start_dictation`/`listen`/`getUserMedia`/ + // `addModule` would let the async start finish against a torn-down + // instance. A fresh startRecording clears this flag before its first + // await, so it never wrongly aborts a subsequent start. + startAbortedRef.current = true; + // Flush any remaining audio before teardown, then stop the native engine. + // Scope the stop to THIS hook instance's session. `cleanup` runs as every + // instance's unmount handler, so an unscoped stop here would let a + // non-recording composer (e.g. a thread reply composer closing) tear down + // the singleton engine owned by another composer that is actively + // recording. `nativeSessionRef.current` is 0 for an instance that never + // started a session (native IDs start at 1), so its stop can never match + // the live session and correctly no-ops. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + // Stop batch timer. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + // Disconnect audio worklet. Clear the port handler first so any PCM + // messages still queued on the main thread are dropped instead of + // appended to the (reused) audio batch. + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + // Drop any audio still buffered so it can't leak into the next session. + audioBatchRef.current = []; + // Close audio context. + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Unlisten events. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + }, [flushAudioBatch]); + + // Cleanup on unmount. + useEffect(() => cleanup, [cleanup]); + + const startRecording = useCallback(async () => { + // Also guard on `isTranscribing`: after `stopRecording()` the previous + // session has cleared `isRecording` but is still awaiting its native + // `stopped` event, which delivers the final transcript before its + // listeners are unregistered. Starting a new session in that window would + // unlisten the old session's handlers (below) before its last words + // arrived, dropping them. Wait for the prior session to fully stop. + if (!isEnabled || isStarting || isRecording || isTranscribing) return; + + // Clear abort flag for this new start attempt. + startAbortedRef.current = false; + // Reset any leftover audio buffer so a stale batch from a prior session + // can't be flushed into this new session/draft. + audioBatchRef.current = []; + // Reset the flush chain so this session's flushes don't chain behind a + // prior session's (already-settled) tail. + flushChainRef.current = Promise.resolve(); + + setIsStarting(true); + + try { + // 1. Start the native STT engine — returns the session ID used to tag events. + const sessionId = await invoke("start_dictation"); + nativeSessionRef.current = sessionId; + + // Bail if aborted during engine start. + if (startAbortedRef.current) { + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // Unregister any lingering listeners from a previous session so they + // can't match the new session ID via the shared ref. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + + // 2. Listen for transcript events from the native layer. + // Each listener captures `sessionId` by value (closure) so it only + // matches events from this specific session — immune to ref mutation. + const unlistenTranscript = await listen<{ + text: string; + session: number; + }>(DICTATION_TRANSCRIPT_EVENT, (event) => { + const { text, session } = event.payload; + if (session !== sessionId) return; + if (text) { + onTranscriptTextRef.current(text); + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenTranscriptRef.current = unlistenTranscript; + + const unlistenState = await listen<{ + state: string; + session: number; + }>(DICTATION_STATE_EVENT, (event) => { + const { state, session } = event.payload; + if (session !== sessionId) return; + if (state === "stopped") { + setIsRecording(false); + setIsTranscribing(false); + // Tear down this instance's local capture pipeline. The native + // engine is a singleton: when another mounted composer calls + // `start_dictation`, it stops this session's engine, so we can + // receive `stopped` without our own `stopRecording()`/`cleanup()` + // having run. Without tearing down here, `streamRef`/`workletRef`/ + // `batchTimerRef` stay alive and this composer keeps the mic open, + // pushing stale audio, until it unmounts. Don't re-invoke + // `stop_dictation` — the native side already stopped (that's why + // this event fired). + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + audioBatchRef.current = []; + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Clean up event listeners now that the session is fully done. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + unlistenState(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenStateRef.current = unlistenState; + + // 3. Capture mic audio. + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }); + streamRef.current = stream; + + // Bail if aborted during mic permission prompt. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // 4. Set up AudioWorklet to send PCM to native layer. + const audioContext = new AudioContext({ sampleRate: 48000 }); + audioContextRef.current = audioContext; + + // Resume if the WebView created the context suspended (autoplay policy). + // A suspended context never pulls the worklet's `process()`, so no PCM + // would reach `push_dictation_audio` while the UI shows an active + // recording. Mirrors the huddle capture path (`huddle/lib/audioWorklet.ts`). + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + // Create a processor that accumulates audio frames and posts them + // to the main thread. Batching happens on the main thread side via + // a timer to reduce IPC overhead. + const processorCode = ` + class DictationProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0] && input[0].length > 0) { + this.port.postMessage(input[0].buffer); + } + return true; + } + } + registerProcessor('dictation-processor', DictationProcessor); + `; + const blob = new Blob([processorCode], { + type: "application/javascript", + }); + const blobUrl = URL.createObjectURL(blob); + try { + await audioContext.audioWorklet.addModule(blobUrl); + } finally { + URL.revokeObjectURL(blobUrl); + } + + // Bail if stop/cancel was called while the worklet module was loading. + // Without this the worklet/flush timer would start and `isRecording` + // would be set true, leaving dictation running after it was stopped. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + void audioContext.close(); + audioContextRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + const source = audioContext.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode(audioContext, "dictation-processor"); + workletRef.current = worklet; + + // Accumulate audio frames in a batch array; a timer flushes to native. + worklet.port.onmessage = (event: MessageEvent) => { + audioBatchRef.current.push(new Float32Array(event.data)); + }; + + // Start the batch flush timer (~10 IPC calls/s instead of ~375). + batchTimerRef.current = setInterval(flushAudioBatch, AUDIO_BATCH_MS); + + const outputGain = audioContext.createGain(); + outputGain.gain.value = 0; + outputGainRef.current = outputGain; + + source.connect(worklet); + worklet.connect(outputGain); + outputGain.connect(audioContext.destination); + + setIsRecording(true); + setIsTranscribing(true); + } catch (error) { + // Tear down and stop the native engine if it was started but a later + // step failed (e.g. mic permission denied, AudioWorklet setup error). + // `cleanup()` performs the session-scoped stop, so it only tears down + // this instance's own session — never another composer's. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + + const message = + error instanceof Error ? error.message : "Local dictation failed"; + if (/not allowed|denied|permission/i.test(message)) { + toast.error("Microphone access denied", { + description: + "Allow microphone access in System Settings to use dictation.", + }); + } else if (/not found|no audio/i.test(message)) { + toast.error("No microphone found", { + description: "Connect a microphone and try again.", + }); + } else if (/model not ready/i.test(message)) { + toast.error("Voice model downloading", { + description: + "The speech model is still downloading. Try again shortly.", + }); + } else { + toast.error("Dictation failed", { description: message }); + } + } finally { + setIsStarting(false); + } + }, [ + cleanup, + flushAudioBatch, + isEnabled, + isRecording, + isStarting, + isTranscribing, + ]); + + const stopRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // Stop batch timer immediately. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic and audio pipeline immediately so the user gets visual feedback. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + // Clear the port handler so PCM messages still queued on the main + // thread are dropped rather than appended to the batch after the + // final flush below (and leaking into the next session). + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + setIsRecording(false); + // Flush remaining audio and THEN stop the native engine, ensuring the + // final batch arrives before the engine shuts down and flushes its buffer. + // isTranscribing stays true — cleared when `dictation-state: stopped` arrives. + // Scope the stop to THIS session: if the user restarts during the flush + // window, a new session may already be stored by the time this resolves — + // passing the session keeps this stop from tearing down the new engine. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + }, [flushAudioBatch]); + + const cancelRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // `cleanup()` performs the session-scoped native stop, so cancelling a + // composer that isn't the active recorder can't tear down another + // composer's session. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + }, [cleanup]); + + const toggleRecording = useCallback(() => { + if (isRecording || isStarting) { + stopRecording(); + return; + } + void startRecording(); + }, [isRecording, isStarting, startRecording, stopRecording]); + + return { + isEnabled, + isRecording, + isStarting, + isTranscribing, + startRecording, + stopRecording, + cancelRecording, + toggleRecording, + }; +} diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx new file mode 100644 index 0000000000..e026c60c30 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -0,0 +1,101 @@ +import type * as React from "react"; +import { useCallback, useEffect, useRef } from "react"; +import { useFeatureEnabled } from "@/shared/features"; +import { + getDictationSendDecision, + shouldAutoSubmitDictation, +} from "../lib/voiceInput"; +import { DictationButton } from "../ui/DictationButton"; +import { useComposerDictation } from "./useComposerDictation"; + +interface UseMessageComposerDictationOptions { + syncContentRef: React.MutableRefObject<() => string>; + disabled: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + setComposerContent: (text: string) => void; + setEditorContent: (text: string) => void; + draftKey: string | null; + composerRef: React.RefObject; + submitMessageRef: React.MutableRefObject<() => void>; +} + +export function useMessageComposerDictation({ + disabled, + draftKey, + setEditorContent, + submitMessageRef, + ...options +}: UseMessageComposerDictationOptions) { + const enabled = useFeatureEnabled("voiceDictation"); + const setEditorContentRef = useRef(setEditorContent); + setEditorContentRef.current = setEditorContent; + const dictation = useComposerDictation({ + ...options, + disabled, + draftKey, + enabled, + setEditorContentRef, + }); + const { isRecording, isStarting, isTranscribing, stopRecording } = dictation; + const sendAfterTranscriptionRef = useRef(false); + + // biome-ignore lint/correctness/useExhaustiveDependencies: composer scope and availability are the reset triggers for a queued send + useEffect(() => { + sendAfterTranscriptionRef.current = false; + }, [disabled, draftKey]); + + useEffect(() => { + if ( + !shouldAutoSubmitDictation({ + requested: sendAfterTranscriptionRef.current, + isRecording, + isStarting, + isTranscribing, + }) + ) { + return; + } + sendAfterTranscriptionRef.current = false; + submitMessageRef.current(); + }, [isRecording, isStarting, isTranscribing, submitMessageRef]); + + const prepareToSubmit = useCallback(() => { + const decision = getDictationSendDecision({ + isRecording, + isStarting, + isTranscribing, + }); + if (decision !== "send") { + sendAfterTranscriptionRef.current = true; + } + if (decision === "stop-recording") { + stopRecording(); + } + return decision === "send"; + }, [isRecording, isStarting, isTranscribing, stopRecording]); + + return { + ...dictation, + canStopFromSend: isRecording || isStarting, + prepareToSubmit, + }; +} + +export function MessageComposerDictationAction({ + children, + dictation, + disabled, +}: { + children?: React.ReactNode; + dictation: ReturnType; + disabled: boolean; +}) { + return ( + <> + + {children} + + ); +} diff --git a/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts b/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts new file mode 100644 index 0000000000..31597ed7ae --- /dev/null +++ b/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts @@ -0,0 +1,53 @@ +import * as React from "react"; +import { useFeatureEnabled } from "@/shared/features"; +import { hasPrimaryShortcutModifier } from "@/shared/lib/platform"; + +export function useVoiceDictationShortcut(disabled: boolean) { + const enabled = useFeatureEnabled("voiceDictation"); + + React.useLayoutEffect(() => { + if (!enabled || disabled) { + return; + } + + let keyHeld = false; + const release = () => { + if (!keyHeld) return; + keyHeld = false; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-up")); + }; + const handleKeyDown = (event: KeyboardEvent) => { + if ( + event.defaultPrevented || + event.repeat || + event.altKey || + event.shiftKey || + event.key.toLowerCase() !== "d" || + !hasPrimaryShortcutModifier(event) + ) { + return; + } + event.preventDefault(); + keyHeld = true; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-down")); + }; + const handleKeyUp = (event: KeyboardEvent) => { + if (event.key.toLowerCase() === "d") release(); + }; + const handleVisibilityChange = () => { + if (document.hidden) release(); + }; + + window.addEventListener("keydown", handleKeyDown); + window.addEventListener("keyup", handleKeyUp); + window.addEventListener("blur", release); + document.addEventListener("visibilitychange", handleVisibilityChange); + return () => { + release(); + window.removeEventListener("keydown", handleKeyDown); + window.removeEventListener("keyup", handleKeyUp); + window.removeEventListener("blur", release); + document.removeEventListener("visibilitychange", handleVisibilityChange); + }; + }, [disabled, enabled]); +} diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts new file mode 100644 index 0000000000..12964f17af --- /dev/null +++ b/desktop/src/features/dictation/index.ts @@ -0,0 +1,8 @@ +export { useComposerDictation } from "./hooks/useComposerDictation"; +export { useDictation } from "./hooks/useDictation"; +export { + MessageComposerDictationAction, + useMessageComposerDictation, +} from "./hooks/useMessageComposerDictation"; +export { useVoiceDictationShortcut } from "./hooks/useVoiceDictationShortcut"; +export { DictationButton } from "./ui/DictationButton"; diff --git a/desktop/src/features/dictation/lib/activeComposer.ts b/desktop/src/features/dictation/lib/activeComposer.ts new file mode 100644 index 0000000000..1fb1b3f104 --- /dev/null +++ b/desktop/src/features/dictation/lib/activeComposer.ts @@ -0,0 +1,23 @@ +/** + * Tracks which composer instance should handle global dictation shortcuts. + * + * When multiple composers are mounted (e.g. channel + thread), only the + * most recently focused one should respond to ⌘D. Each composer registers + * on focus and the global shortcut only dispatches to the active instance. + */ + +let activeInstanceId: string | null = null; + +export function setActiveDictationComposer(id: string): void { + activeInstanceId = id; +} + +export function clearActiveDictationComposer(id: string): void { + if (activeInstanceId === id) { + activeInstanceId = null; + } +} + +export function isActiveDictationComposer(id: string): boolean { + return activeInstanceId === id; +} diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs new file mode 100644 index 0000000000..2ae61b3387 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + appendTranscribedText, + getDictationSendDecision, + shouldAutoSubmitDictation, +} from "./voiceInput.ts"; + +test("dictation send stops capture before submitting", () => { + assert.equal( + getDictationSendDecision({ + isRecording: true, + isStarting: false, + isTranscribing: true, + }), + "stop-recording", + ); + assert.equal( + getDictationSendDecision({ + isRecording: false, + isStarting: true, + isTranscribing: false, + }), + "stop-recording", + ); +}); + +test("dictation send waits for the final transcript", () => { + assert.equal( + getDictationSendDecision({ + isRecording: false, + isStarting: false, + isTranscribing: true, + }), + "wait", + ); + assert.equal( + getDictationSendDecision({ + isRecording: false, + isStarting: false, + isTranscribing: false, + }), + "send", + ); +}); + +test("dictation auto-submit waits until capture and transcription settle", () => { + const state = { + requested: true, + isRecording: false, + isStarting: false, + isTranscribing: false, + }; + assert.equal(shouldAutoSubmitDictation(state), true); + assert.equal( + shouldAutoSubmitDictation({ ...state, requested: false }), + false, + ); + assert.equal( + shouldAutoSubmitDictation({ ...state, isRecording: true }), + false, + ); + assert.equal( + shouldAutoSubmitDictation({ ...state, isStarting: true }), + false, + ); + assert.equal( + shouldAutoSubmitDictation({ ...state, isTranscribing: true }), + false, + ); +}); + +// ── appendTranscribedText ─────────────────────────────────────────────────── + +test("appendTranscribedText_appendsToExistingText", () => { + assert.equal(appendTranscribedText("Hello", "world"), "Hello world"); +}); + +test("appendTranscribedText_appendsToEmptyBase", () => { + assert.equal(appendTranscribedText("", "hello"), "hello"); +}); + +test("appendTranscribedText_avoidsSpaceBeforePunctuation", () => { + assert.equal(appendTranscribedText("Hello", ", world"), "Hello, world"); +}); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts new file mode 100644 index 0000000000..7bb10c3a9f --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -0,0 +1,42 @@ +export function appendTranscribedText( + baseText: string, + fragment: string, +): string { + const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); + if (!normalizedFragment) return baseText; + if (!baseText.trim()) return normalizedFragment; + if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) { + return `${baseText}${normalizedFragment}`; + } + return `${baseText} ${normalizedFragment}`; +} + +export type DictationSendDecision = "send" | "stop-recording" | "wait"; + +export function getDictationSendDecision({ + isRecording, + isStarting, + isTranscribing, +}: { + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; +}): DictationSendDecision { + if (isRecording || isStarting) return "stop-recording"; + if (isTranscribing) return "wait"; + return "send"; +} + +export function shouldAutoSubmitDictation({ + requested, + isRecording, + isStarting, + isTranscribing, +}: { + requested: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; +}): boolean { + return requested && !isRecording && !isStarting && !isTranscribing; +} diff --git a/desktop/src/features/dictation/ui/DictationButton.tsx b/desktop/src/features/dictation/ui/DictationButton.tsx new file mode 100644 index 0000000000..f284957e17 --- /dev/null +++ b/desktop/src/features/dictation/ui/DictationButton.tsx @@ -0,0 +1,73 @@ +import { Mic, Square } from "lucide-react"; +import { Button } from "@/shared/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; +import { cn } from "@/shared/lib/cn"; +import { isMacPlatform } from "@/shared/lib/platform"; + +interface DictationState { + isEnabled: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; + toggleRecording: () => void; +} + +interface DictationButtonProps { + dictation: DictationState; + disabled?: boolean; +} + +export function DictationButton({ + dictation, + disabled = false, +}: DictationButtonProps) { + if (!dictation.isEnabled) return null; + + const shortcutHint = isMacPlatform() ? "⌘D" : "Ctrl+D"; + + const tooltipText = dictation.isRecording + ? "Stop recording" + : dictation.isTranscribing + ? "Transcribing…" + : null; + + return ( + + + + + + {tooltipText ?? ( + + Voice Dictation + + {shortcutHint} + + + )} + + + ); +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 69e4ec67b5..ce9b6f8ee5 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -2,9 +2,9 @@ import * as React from "react"; import { EditorContent } from "@tiptap/react"; import { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; +import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import { handleAgentSnapshotPaste } from "@/features/messages/lib/agentSnapshotClipboard"; import { useComposerAutofocus } from "@/features/messages/lib/useComposerAutofocus"; -import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import { useDrafts } from "@/features/messages/lib/useDrafts"; import { resolveSentDraftKey } from "@/features/messages/ui/draftSubmitKey"; import { useEmojiAutocomplete } from "@/features/messages/lib/useEmojiAutocomplete"; @@ -39,22 +39,26 @@ import { import { useLinkEditor } from "@/features/messages/lib/useLinkEditor"; import { useComposerSpoilerParticles } from "@/features/messages/lib/useComposerSpoilerParticles"; import { useTypingBroadcast } from "@/features/messages/useTypingBroadcast"; +import { + MessageComposerDictationAction, + useMessageComposerDictation, +} from "@/features/dictation"; import { getBuzzCodeBlockClipboardText } from "@/shared/lib/codeBlockClipboard"; import { cn } from "@/shared/lib/cn"; -import { ChannelAutocomplete } from "./ChannelAutocomplete"; import { ComposerReplyEditBanner } from "./ComposerReplyEditBanner"; import { ComposerAttachments, DropZoneOverlay } from "./ComposerAttachments"; -import { EmojiAutocomplete } from "./EmojiAutocomplete"; -import { - MentionAutocomplete, - type MentionSuggestion, -} from "./MentionAutocomplete"; +import type { MentionSuggestion } from "./MentionAutocomplete"; import { ComposerDockToolbar } from "./ComposerDockToolbar"; import { NonMemberMentionDialog } from "./NonMemberMentionDialog"; import { useMentionSendFlow } from "./useMentionSendFlow"; import { usePersistentAgentMentionHydration } from "./usePersistentAgentMentionHydration"; import { useComposerContentState } from "./useComposerContentState"; import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; +import { + MessageComposerAutocompletes, + MessageComposerUploadError, + useComposerScrollToBottom, +} from "./MessageComposerOverlays"; import type { MessageComposerProps } from "./MessageComposer.types"; @@ -206,6 +210,7 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); + const prepareDictationSubmitRef = React.useRef<() => boolean>(() => true); const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler @@ -219,13 +224,7 @@ function MessageComposerImpl({ >(null); const onLinkShortcutRef = React.useRef<(() => boolean) | null>(null); - const scrollComposerToBottom = React.useCallback(() => { - window.requestAnimationFrame(() => { - const scrollElement = composerScrollRef.current; - if (!scrollElement) return; - scrollElement.scrollTop = scrollElement.scrollHeight; - }); - }, []); + const scrollComposerToBottom = useComposerScrollToBottom(composerScrollRef); const computedPlaceholder = editTarget ? "Edit your message" @@ -233,7 +232,6 @@ function MessageComposerImpl({ (replyTarget ? `Reply to ${replyTarget.author} in #${channelName}` : `Message #${channelName}`)); - const richText = useRichTextEditor({ placeholder: computedPlaceholder, editable: !disabled, @@ -268,6 +266,19 @@ function MessageComposerImpl({ }, }); + const dictation = useMessageComposerDictation({ + syncContentRef: syncContentRefFromEditorRef, + disabled, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContent: richText.setContent, + draftKey: `${effectiveDraftKey}\0${editTarget?.id ?? ""}\0${replyTarget?.id ?? ""}`, + composerRef: composerScrollRef, + submitMessageRef, + }); + prepareDictationSubmitRef.current = dictation.prepareToSubmit; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); @@ -278,7 +289,6 @@ function MessageComposerImpl({ onLinkSelectionChangeRef.current = linkEditor.showFromCursor; onLinkShortcutRef.current = linkEditor.openFromShortcut; useComposerSpoilerParticles(richText.editor, composerScrollRef); - const persistentMentionHydration = usePersistentAgentMentionHydration({ audienceScope, hydrationKey: effectiveDraftKey, @@ -507,6 +517,7 @@ function MessageComposerImpl({ // ── Submit message ────────────────────────────────────────────────── const submitMessage = React.useCallback(async () => { + if (!prepareDictationSubmitRef.current()) return; const trimmed = syncComposerContentFromEditor().trim(); // Edit mode @@ -587,7 +598,6 @@ function MessageComposerImpl({ ) { return; } - const capturedThreadContext = onCaptureSendContext?.() ?? null; if ( capturedThreadContext !== null && @@ -827,13 +837,16 @@ function MessageComposerImpl({ disabled || media.isUploading || mentionSendFlow.isPreparingMentionSend || - (isContentEmpty && media.pendingImeta.length === 0), + (isContentEmpty && + media.pendingImeta.length === 0 && + !dictation.canStopFromSend), [ disabled, media.isUploading, mentionSendFlow.isPreparingMentionSend, isContentEmpty, media.pendingImeta.length, + dictation.canStopFromSend, ], ); @@ -919,42 +932,22 @@ function MessageComposerImpl({ }} > {ownsDropZone && media.isDragOver && } - - media.setUploadState({ status: "idle" })} /> - - {media.uploadState.status === "error" ? ( -
- Upload failed: {media.uploadState.message} - -
- ) : null} {(media.pendingImeta.length > 0 || media.isUploading) && (
@@ -988,7 +981,14 @@ function MessageComposerImpl({ layoutMode={layoutMode} composerDisabled={disabled} editor={richText.editor} - extraActions={toolbarExtraActions} + extraActions={ + + {toolbarExtraActions} + + } formattingDisabled={disabled} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen} diff --git a/desktop/src/features/messages/ui/MessageComposerOverlays.tsx b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx new file mode 100644 index 0000000000..cf2b5fde4a --- /dev/null +++ b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx @@ -0,0 +1,85 @@ +import * as React from "react"; +import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; +import type { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; +import type { EmojiSuggestion } from "@/features/messages/lib/useEmojiAutocomplete"; +import type { useEmojiAutocomplete } from "@/features/messages/lib/useEmojiAutocomplete"; +import type { useMentions } from "@/features/messages/lib/useMentions"; +import { ChannelAutocomplete } from "./ChannelAutocomplete"; +import { EmojiAutocomplete } from "./EmojiAutocomplete"; +import { + MentionAutocomplete, + type MentionSuggestion, +} from "./MentionAutocomplete"; + +export function useComposerScrollToBottom( + composerScrollRef: React.RefObject, +) { + return React.useCallback(() => { + window.requestAnimationFrame(() => { + const scrollElement = composerScrollRef.current; + if (!scrollElement) return; + scrollElement.scrollTop = scrollElement.scrollHeight; + }); + }, [composerScrollRef]); +} + +export function MessageComposerAutocompletes({ + applyChannelInsert, + applyEmojiInsert, + applyMentionInsert, + channelLinks, + emojiAutocomplete, + mentions, +}: { + applyChannelInsert: (suggestion: ChannelSuggestion) => void; + applyEmojiInsert: (suggestion: EmojiSuggestion) => void; + applyMentionInsert: (suggestion: MentionSuggestion) => void; + channelLinks: ReturnType; + emojiAutocomplete: ReturnType; + mentions: ReturnType; +}) { + return ( + <> + + + + + ); +} + +export function MessageComposerUploadError({ + message, + onDismiss, +}: { + message: string | null | undefined; + onDismiss: () => void; +}) { + if (message == null) return null; + return ( +
+ Upload failed: {message} + +
+ ); +} diff --git a/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx b/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx index c71e62d5d9..af7bfafbb3 100644 --- a/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx +++ b/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx @@ -3,6 +3,7 @@ import { getPlatformKeys, type KeyboardShortcut, } from "@/shared/lib/keyboard-shortcuts"; +import { useFeatureEnabled } from "@/shared/features"; import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; @@ -29,6 +30,7 @@ function KeyCombo({ shortcut }: { shortcut: KeyboardShortcut }) { } export function KeyboardShortcutsCard() { + const voiceDictationEnabled = useFeatureEnabled("voiceDictation"); const categories = getShortcutsByCategory(); return ( @@ -45,22 +47,27 @@ export function KeyboardShortcutsCard() { {category} - {shortcuts.map((shortcut) => ( - -
- - {shortcut.label} - - - {shortcut.description} - -
- -
- ))} + {shortcuts + .filter( + (shortcut) => + voiceDictationEnabled || shortcut.id !== "voice-dictation", + ) + .map((shortcut) => ( + +
+ + {shortcut.label} + + + {shortcut.description} + +
+ +
+ ))}
))} diff --git a/desktop/src/shared/lib/keyboard-shortcuts.ts b/desktop/src/shared/lib/keyboard-shortcuts.ts index 422c45c5f3..377e4ae616 100644 --- a/desktop/src/shared/lib/keyboard-shortcuts.ts +++ b/desktop/src/shared/lib/keyboard-shortcuts.ts @@ -181,6 +181,14 @@ export const KEYBOARD_SHORTCUTS: KeyboardShortcut[] = [ keysWindows: "Ctrl+Space", category: "Messages", }, + { + id: "voice-dictation", + label: "Voice Dictation", + description: "Hold to dictate, release to stop (push-to-talk)", + keys: "⌘D", + keysWindows: "Ctrl+D", + category: "Messages", + }, // Formatting { diff --git a/desktop/tests/e2e/messaging.spec.ts b/desktop/tests/e2e/messaging.spec.ts index c6f5aefb9b..3e6a0ae40c 100644 --- a/desktop/tests/e2e/messaging.spec.ts +++ b/desktop/tests/e2e/messaging.spec.ts @@ -109,7 +109,72 @@ test.beforeEach(async ({ page }, testInfo) => { ], } : undefined; - await installMockBridge(page, mock); + await installMockBridge(page, mock, { + seedPreviewFeatures: !testInfo.title.includes("dictation experiment"), + }); +}); + +test("voice dictation stays behind the dictation experiment", async ({ + page, +}) => { + await page.goto("/"); + await page.getByTestId("channel-general").click(); + + await page.evaluate(() => { + let dictationStarts = 0; + window.addEventListener("buzz:dictation-key-down", () => { + dictationStarts += 1; + }); + ( + window as typeof window & { __BUZZ_DICTATION_STARTS__?: () => number } + ).__BUZZ_DICTATION_STARTS__ = () => dictationStarts; + }); + + const dispatchDictationShortcut = () => + page.evaluate(() => { + const isMac = /mac|iphone|ipad|ipod/i.test(navigator.platform); + window.dispatchEvent( + new KeyboardEvent("keydown", { + bubbles: true, + ctrlKey: !isMac, + key: "d", + metaKey: isMac, + }), + ); + window.dispatchEvent( + new KeyboardEvent("keyup", { + bubbles: true, + ctrlKey: !isMac, + key: "d", + metaKey: isMac, + }), + ); + }); + const getDictationStarts = () => + page.evaluate( + () => + ( + window as typeof window & { + __BUZZ_DICTATION_STARTS__?: () => number; + } + ).__BUZZ_DICTATION_STARTS__?.() ?? 0, + ); + + await page.getByTestId("message-input").click(); + await dispatchDictationShortcut(); + await expect.poll(getDictationStarts).toBe(0); + + await openSettings(page, "experimental"); + const dictationToggle = page.getByTestId("feature-toggle-voiceDictation"); + await expect(dictationToggle).not.toBeChecked(); + await dictationToggle.click(); + await expect(dictationToggle).toBeChecked(); + + await page.getByTestId("settings-back-to-app").click(); + await expect(page.getByTestId("message-input")).toBeVisible(); + await page.getByTestId("message-input").click(); + await dispatchDictationShortcut(); + await expect.poll(getDictationStarts).toBe(1); }); test("agent owner label identifies the agent and owner", async ({ page }) => { diff --git a/preview-features.json b/preview-features.json index 388f1c39b0..cfd37ee873 100644 --- a/preview-features.json +++ b/preview-features.json @@ -30,6 +30,12 @@ "name": "Agent-managed profiles", "description": "Let agents manage their own relay name and avatar instead of restoring the desktop copy", "platforms": ["desktop"] + }, + { + "id": "voiceDictation", + "name": "Voice dictation", + "description": "Compose messages with local, real-time speech recognition", + "platforms": ["desktop"] } ] }