From e7fe063a9abefeee90018cd7c1d983db32554276 Mon Sep 17 00:00:00 2001 From: Chris233 Date: Wed, 5 Aug 2026 22:56:22 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(pipeline):=20=E5=A4=9A=E6=A8=A1?= =?UTF-8?q?=E6=80=81=E8=AF=86=E5=88=AB=E7=AE=A1=E7=BA=BF=EF=BC=88=E5=AE=9E?= =?UTF-8?q?=E9=AA=8C=E6=80=A7=EF=BC=89=EF=BC=8C=E4=BC=A0=E7=BB=9F/?= =?UTF-8?q?=E5=A4=9A=E6=A8=A1=E6=80=81=E6=A8=A1=E5=BC=8F=E4=B8=8E=20omni?= =?UTF-8?q?=20=E5=87=AD=E6=8D=AE=E9=9A=94=E7=A6=BB=20(#902)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增实验性「多模态识别管线」：高级设置开启后，服务页出现传统/多模态模式切换。多模态模式用单个模型（OpenAI 兼容 input_audio 或 Gemini 原生 generateContent）一步完成「提示词 + 音频 → 最终文本」，覆盖主听写、划词问答（OpenAI 兼容流式 / Gemini 一次性）、选区润色（omni 当纯文本 LLM）、Less Computer 转写。两套配置在凭据库中完全隔离（新增 omni 命名空间），切换不删数据、不回退传统配置。 --- .../app/src-tauri/src/commands/credentials.rs | 52 +- .../app/src-tauri/src/commands/history.rs | 29 +- .../app/src-tauri/src/commands/providers.rs | 47 +- openless-all/app/src-tauri/src/coordinator.rs | 285 ++-- .../src-tauri/src/coordinator/dictation.rs | 626 ++++++-- .../src-tauri/src/coordinator/polish_flow.rs | 878 +++++------ .../src-tauri/src/coordinator/qa_session.rs | 189 ++- .../src-tauri/src/coordinator/resources.rs | 68 + .../src/coordinator/selection_polish.rs | 19 +- openless-all/app/src-tauri/src/lib.rs | 69 +- openless-all/app/src-tauri/src/llm_gemini.rs | 44 + openless-all/app/src-tauri/src/omni.rs | 481 ++++++ .../src-tauri/src/persistence/credentials.rs | 273 +++- openless-all/app/src-tauri/src/polish.rs | 74 +- openless-all/app/src-tauri/src/types.rs | 88 +- openless-all/app/src/i18n/en.ts | 15 + openless-all/app/src/i18n/ja.ts | 15 + openless-all/app/src/i18n/ko.ts | 15 + openless-all/app/src/i18n/zh-CN.ts | 15 + openless-all/app/src/i18n/zh-TW.ts | 15 + .../app/src/lib/ipc/asr-credentials.ts | 108 +- openless-all/app/src/lib/ipc/index.ts | 1 + openless-all/app/src/lib/ipc/mock-data.ts | 5 + .../app/src/lib/providerSetup.test.ts | 168 ++- openless-all/app/src/lib/providerSetup.ts | 26 +- openless-all/app/src/lib/stylePrefs.test.ts | 3 + openless-all/app/src/lib/types.ts | 17 + openless-all/app/src/pages/History.tsx | 1302 +++++++++-------- openless-all/app/src/pages/Overview.tsx | 2 + .../settings/MultimodalPipelineSection.tsx | 42 + .../src/pages/settings/ProvidersSection.tsx | 213 ++- openless-all/app/src/pages/settings/tabs.tsx | 2 + 32 files changed, 3655 insertions(+), 1531 deletions(-) create mode 100644 openless-all/app/src-tauri/src/omni.rs create mode 100644 openless-all/app/src/pages/settings/MultimodalPipelineSection.tsx diff --git a/openless-all/app/src-tauri/src/commands/credentials.rs b/openless-all/app/src-tauri/src/commands/credentials.rs index 5030e60e9..a8e82d3db 100644 --- a/openless-all/app/src-tauri/src/commands/credentials.rs +++ b/openless-all/app/src-tauri/src/commands/credentials.rs @@ -2,6 +2,8 @@ use super::*; const LLM_EXTRA_HEADERS_ACCOUNT: &str = "ark.extra_headers"; const LLM_TEMPERATURE_ACCOUNT: &str = "ark.temperature"; +const OMNI_EXTRA_HEADERS_ACCOUNT: &str = "omni.extra_headers"; +const OMNI_TEMPERATURE_ACCOUNT: &str = "omni.temperature"; #[tauri::command] pub async fn get_credentials() -> Result { @@ -9,14 +11,20 @@ pub async fn get_credentials() -> Result { let snap = CredentialsVault::snapshot(); let active_asr_provider = CredentialsVault::get_active_asr(); let active_llm_provider = CredentialsVault::get_active_llm(); + let pipeline_mode = PreferencesStore::new() + .map(|store| store.get().pipeline_mode) + .unwrap_or(crate::types::PipelineMode::Traditional); let volcengine_configured = volcengine_configured(&snap); let asr_configured = asr_configured_for_provider(&active_asr_provider, &snap); let llm_configured = llm_configured_for_provider(&active_llm_provider, &snap); + let omni_configured = omni_configured_for_active_provider(&snap); CredentialsStatus { active_asr_provider, active_llm_provider, + pipeline_mode, asr_configured, llm_configured, + omni_configured, volcengine_configured, ark_configured: llm_configured, } @@ -136,6 +144,18 @@ fn configured(field: &Option) -> bool { .unwrap_or(false) } +/// 多模态（Omni）模型是否已配置：OpenAI 兼容通道要求 API Key + Base URL + Model； +/// Gemini 通道要求 API Key + Model（Base URL 为空时后端走官方默认）。 +pub(crate) fn omni_configured_for_active_provider(snap: &CredentialsSnapshot) -> bool { + let provider = &snap.active_omni_provider; + let has_api_key = configured(&snap.omni_api_key); + let has_model = configured(&snap.omni_model); + if provider == "gemini" { + return has_api_key && has_model; + } + has_api_key && configured(&snap.omni_endpoint) && has_model +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg(not(mobile))] pub(crate) struct LocalAsrReleasePlan { @@ -189,7 +209,9 @@ pub async fn set_credential( ensure_main_window(&window)?; let extra_headers = account == LLM_EXTRA_HEADERS_ACCOUNT; let temperature = account == LLM_TEMPERATURE_ACCOUNT; - let parsed = if extra_headers || temperature { + let omni_extra_headers = account == OMNI_EXTRA_HEADERS_ACCOUNT; + let omni_temperature = account == OMNI_TEMPERATURE_ACCOUNT; + let parsed = if extra_headers || temperature || omni_extra_headers || omni_temperature { None } else { Some(parse_account(&account)?) @@ -200,7 +222,14 @@ pub async fn set_credential( .map_err(|e| e.to_string()); } if temperature { - return CredentialsVault::set_active_llm_temperature(&value) + return CredentialsVault::set_active_llm_temperature(&value).map_err(|e| e.to_string()); + } + if omni_extra_headers { + return CredentialsVault::set_active_omni_extra_headers_json(&value) + .map_err(|e| e.to_string()); + } + if omni_temperature { + return CredentialsVault::set_active_omni_temperature(&value) .map_err(|e| e.to_string()); } let acc = parsed.expect("non-extra credential account must be parsed"); @@ -304,6 +333,11 @@ pub fn set_active_llm_provider(provider: String) -> Result<(), String> { CredentialsVault::set_active_llm_provider(&provider).map_err(|e| e.to_string()) } +#[tauri::command] +pub fn set_active_omni_provider(provider: String) -> Result<(), String> { + CredentialsVault::set_active_omni_provider(&provider).map_err(|e| e.to_string()) +} + /// 读出某个账号的实际值（用于设置页预填表单）。 /// 凭据来自系统凭据库；只允许主设置窗口读取 raw secret，避免胶囊 / QA 等辅助窗口默认暴露。 #[tauri::command] @@ -315,7 +349,9 @@ pub async fn read_credential( ensure_main_window(&window)?; let extra_headers = account == LLM_EXTRA_HEADERS_ACCOUNT; let temperature = account == LLM_TEMPERATURE_ACCOUNT; - let parsed = if extra_headers || temperature { + let omni_extra_headers = account == OMNI_EXTRA_HEADERS_ACCOUNT; + let omni_temperature = account == OMNI_TEMPERATURE_ACCOUNT; + let parsed = if extra_headers || temperature || omni_extra_headers || omni_temperature { None } else { Some(parse_account(&account)?) @@ -328,6 +364,13 @@ pub async fn read_credential( if temperature { return Ok(CredentialsVault::get_active_llm_temperature_string()); } + if omni_extra_headers { + return CredentialsVault::get_active_omni_extra_headers_json() + .map_err(|e| e.to_string()); + } + if omni_temperature { + return Ok(CredentialsVault::get_active_omni_temperature_string()); + } let acc = parsed.expect("non-extra credential account must be parsed"); if let Some(provider) = provider { CredentialsVault::get_for_asr_provider(&provider, acc).map_err(|e| e.to_string()) @@ -364,6 +407,9 @@ fn parse_account(s: &str) -> Result { "asr.advanced_config" => Ok(CredentialAccount::AsrAdvancedConfig), "xfyun.app_id" => Ok(CredentialAccount::XfyunAppId), "xfyun.api_key" => Ok(CredentialAccount::XfyunApiKey), + "omni.api_key" => Ok(CredentialAccount::OmniApiKey), + "omni.endpoint" => Ok(CredentialAccount::OmniEndpoint), + "omni.model" => Ok(CredentialAccount::OmniModel), _ => Err(format!("unknown account: {s}")), } } diff --git a/openless-all/app/src-tauri/src/commands/history.rs b/openless-all/app/src-tauri/src/commands/history.rs index 95c2fed23..ca8cdad0d 100644 --- a/openless-all/app/src-tauri/src/commands/history.rs +++ b/openless-all/app/src-tauri/src/commands/history.rs @@ -60,10 +60,17 @@ pub async fn read_audio_recording(session_id: String) -> Result format!("read wav failed: {e}") } })?; - log::info!("[history] read_audio_recording id={session_id} bytes={} head={:?}", data.len(), &data.get(..16)); + log::info!( + "[history] read_audio_recording id={session_id} bytes={} head={:?}", + data.len(), + &data.get(..16) + ); let b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &data); let data_url = format!("data:audio/wav;base64,{b64}"); - log::info!("[history] read_audio_recording data_url_len={}", data_url.len()); + log::info!( + "[history] read_audio_recording data_url_len={}", + data_url.len() + ); Ok(data_url) } @@ -122,9 +129,7 @@ fn export_recording_to_destination( } } - let destination = file_path - .into_path() - .map_err(export_recording_failed)?; + let destination = file_path.into_path().map_err(export_recording_failed)?; copy_recording_to_path(source, &destination)?; Ok(destination.to_string_lossy().into_owned()) } @@ -151,7 +156,8 @@ fn copy_recording_to_path( destination: &std::path::Path, ) -> Result<(), String> { let mut source_file = open_recording_source(source)?; - let mut destination_file = std::fs::File::create(destination).map_err(export_recording_failed)?; + let mut destination_file = + std::fs::File::create(destination).map_err(export_recording_failed)?; std::io::copy(&mut source_file, &mut destination_file) .map(|_| ()) .map_err(export_recording_failed) @@ -172,7 +178,9 @@ fn copy_recording_to_mobile_url( Ok(file) => file, Err(error) => { #[cfg(target_os = "ios")] - let _ = app.fs().stop_accessing_security_scoped_resource(destination.clone()); + let _ = app + .fs() + .stop_accessing_security_scoped_resource(destination.clone()); return Err(export_recording_failed(error)); } }; @@ -256,7 +264,6 @@ pub async fn retranscribe_recording( Ok(entry) } - /// 把一次重转录的结果落到既有历史条目上（纯函数，供单测覆盖契约）： /// - 只更新转写结果并清除失败标记。insert_status 保持原值——重新转录不向光标落字， /// 没有可表达「已转写未落字」的状态，清掉 error_code 即足以标记不再是失败条目。 @@ -307,6 +314,7 @@ mod retranscribe_tests { asr_model: Some("volc.seedasr.sauc.duration".into()), llm_provider: Some("ark".into()), llm_model: Some("deepseek-v3-2".into()), + pipeline_mode: None, asr_ms: Some(15000), polish_ms: Some(1200), } @@ -325,7 +333,10 @@ mod retranscribe_tests { assert_eq!(entry.final_text, "重转出来的文本"); assert_eq!(entry.error_code, None, "重转成功应清除失败标记"); // ASR 归因换成本次重转的构建时快照。 - assert_eq!(entry.asr_provider.as_deref(), Some("bailian-qwen3-realtime")); + assert_eq!( + entry.asr_provider.as_deref(), + Some("bailian-qwen3-realtime") + ); assert_eq!(entry.asr_model.as_deref(), Some("qwen3-asr-flash-realtime")); assert_eq!(entry.asr_ms, Some(480)); // 重转没有润色环节：旧 LLM 元数据不得残留在新转写结果上。 diff --git a/openless-all/app/src-tauri/src/commands/providers.rs b/openless-all/app/src-tauri/src/commands/providers.rs index 537c304a9..3a600edae 100644 --- a/openless-all/app/src-tauri/src/commands/providers.rs +++ b/openless-all/app/src-tauri/src/commands/providers.rs @@ -22,6 +22,9 @@ pub async fn validate_provider_credentials(kind: String) -> Result validate_asr_provider() .await .map(|()| ProviderCheckResult { ok: true }), + "omni" => validate_omni_provider() + .await + .map(|()| ProviderCheckResult { ok: true }), _ => Err(format!("unknown provider kind: {kind}")), } } @@ -129,6 +132,12 @@ fn read_openai_provider_config(kind: &str) -> Result { CredentialsVault::get_active_asr() != crate::coordinator::OPENAI_COMPATIBLE_ASR_PROVIDER_ID, ), + // 多模态（Omni）模型：独立命名空间，OpenAI 兼容通道要求 API Key + Base URL。 + "omni" => ( + CredentialAccount::OmniApiKey, + CredentialAccount::OmniEndpoint, + true, + ), _ => return Err(format!("unknown provider kind: {kind}")), }; let api_key = CredentialsVault::get(api_key_account) @@ -146,6 +155,15 @@ fn read_openai_provider_config(kind: &str) -> Result { CredentialsVault::get_active_llm_temperature(), ), ) + } else if kind == "omni" { + let active_omni = CredentialsVault::get_active_omni(); + ( + CredentialsVault::get_active_omni_extra_headers(), + openai_compatible_temperature_for_provider( + &active_omni, + CredentialsVault::get_active_omni_temperature(), + ), + ) } else { (HashMap::new(), None) }; @@ -246,6 +264,18 @@ fn provider_llm_error_message(error: LLMError) -> String { } } +/// 多模态（Omni）模型连通性验证：真发一次纯文本请求（无音频），走与运行期 +/// 完全相同的 provider 构建与请求路径，避免「验证通过但真实调用失败」。 +async fn validate_omni_provider() -> Result<(), String> { + let provider = + crate::coordinator::build_active_omni_provider(false).map_err(|e| e.to_string())?; + provider + .complete("验证连接", "ping", None) + .await + .map(|_| ()) + .map_err(provider_llm_error_message) +} + async fn validate_asr_provider() -> Result<(), String> { let active_asr = CredentialsVault::get_active_asr(); if active_asr_is_keyless_for_validation(&active_asr) { @@ -746,8 +776,7 @@ async fn validate_asr_transcription( request.json(&body) } }; - match request.send().await - { + match request.send().await { Ok(resp) => break resp, Err(e) if e.is_timeout() => return Err("providerRequestTimeout".to_string()), Err(e) if (e.is_connect() || e.is_request()) && attempt < MAX_ATTEMPTS => { @@ -1241,9 +1270,12 @@ mod tests { stream.write_all(response.as_bytes()).await.unwrap(); }); let target_server = tokio::spawn(async move { - tokio::time::timeout(std::time::Duration::from_millis(500), target_listener.accept()) - .await - .is_ok() + tokio::time::timeout( + std::time::Duration::from_millis(500), + target_listener.accept(), + ) + .await + .is_ok() }); let error = send_dashscope_multimodal_validation( @@ -1256,7 +1288,10 @@ mod tests { redirect_server.await.unwrap(); assert_eq!(error, "providerHttpStatus:302"); - assert!(!target_server.await.unwrap(), "validation followed redirect"); + assert!( + !target_server.await.unwrap(), + "validation followed redirect" + ); } #[test] diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 6915be840..b97f7b104 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -72,9 +72,9 @@ mod polish_flow; mod qa; mod qa_session; mod resources; -mod silence_auto_stop; #[cfg(not(mobile))] pub(crate) mod selection_polish; +mod silence_auto_stop; use asr_wiring::*; // providers.rs 的 ASR 验证路径按 provider 的真实请求格式发送探针（issue #837）， @@ -113,13 +113,7 @@ use qa::{ }; #[cfg(test)] use resources::discard_startup_resources_for_session; -use resources::{ - acquire_recording_mute, cancel_active_asr, cancel_qa_asr_for_session, release_recording_mute, - selected_microphone_device_name, stop_microphone_preview_monitor, - stop_qa_recorder_for_session, store_qa_asr_for_session, store_qa_recorder_for_session, - take_asr_for_session, take_qa_asr_for_session, take_recorder_for_session, SessionResource, - SharedRecordingMuteState, -}; +use resources::{cancel_active_asr, SessionResource, SharedRecordingMuteState}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum CapsuleShowStrategy { @@ -292,17 +286,16 @@ impl ActiveAsrProviderKind { match self { ActiveAsrProviderKind::Bailian | ActiveAsrProviderKind::Qwen3Realtime - | ActiveAsrProviderKind::ElevenLabs => { - AsrConfiguredFields::ApiKeyOnly - } + | ActiveAsrProviderKind::ElevenLabs => AsrConfiguredFields::ApiKeyOnly, ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal => { AsrConfiguredFields::ApiKeyEndpointModel } // StepfunRealtime 只经 `stepfun` 的模型路由可达（隐藏 effective id）， // 「已配置」判定看真实 active `stepfun` → WhisperCompatible；此处形态 // 与之对齐，保证直接停在该 id 上也语义一致。 - ActiveAsrProviderKind::WhisperCompatible - | ActiveAsrProviderKind::StepfunRealtime => AsrConfiguredFields::EndpointModelOnly, + ActiveAsrProviderKind::WhisperCompatible | ActiveAsrProviderKind::StepfunRealtime => { + AsrConfiguredFields::EndpointModelOnly + } ActiveAsrProviderKind::Volcengine => AsrConfiguredFields::VolcAppKey, ActiveAsrProviderKind::Xfyun => AsrConfiguredFields::XfyunAppKey, } @@ -532,12 +525,10 @@ fn advanced_asr_config_for(provider_id: &str, raw: Option<&str>) -> AdvancedAsrC /// 读取某 ASR provider 的高级配置。仅 `openai-compatible` / `zenmux` 读 vault； /// 其余命名厂商走硬编码行为（这里返回默认值），避免破坏已测通的路径。 fn read_advanced_asr_config(provider_id: &str) -> AdvancedAsrConfig { - let raw = CredentialsVault::get_for_asr_provider( - provider_id, - CredentialAccount::AsrAdvancedConfig, - ) - .ok() - .flatten(); + let raw = + CredentialsVault::get_for_asr_provider(provider_id, CredentialAccount::AsrAdvancedConfig) + .ok() + .flatten(); advanced_asr_config_for(provider_id, raw.as_deref()) } @@ -565,6 +556,10 @@ struct Inner { /// store_asr_for_session 一并写入，end_session 取走落 history——比事后重读 /// 全局设置可靠：会话中途切 provider/model 不会污染归因（PR #826 review）。 asr_label: Mutex>>, + /// 多模态（Omni）模式下的 dictation 录音 PCM 缓冲。只在 + /// `multimodal_pipeline_enabled && pipeline_mode == multimodal` 时使用， + /// 与 asr 槽互斥——同一会话二者有且仅有一个。 + omni_pcm: Mutex>>>, /// 本地 Qwen3-ASR 引擎缓存。跨会话复用，避免每次重加载 1.2GB+ 模型。 /// 释放时机由 prefs.local_asr_keep_loaded_secs 决定。 local_asr_cache: Arc, @@ -667,6 +662,8 @@ struct Inner { capsule_cursor_passthrough: AtomicBool, /// QA 用的 ASR 句柄。必须跟 active_asr_provider 保持一致，避免浮窗走不同入口。 qa_asr: Mutex>>, + /// QA 用的多模态（Omni）录音 PCM 缓冲。与 qa_asr 互斥。 + qa_omni_pcm: Mutex>>>, /// QA 用的 Recorder 句柄。 qa_recorder: Mutex>>, /// QA SSE 流取消标志。begin_qa_session 重置为 false；cancel_qa_session 设 true； @@ -827,6 +824,7 @@ impl Coordinator { state: Mutex::new(SessionState::default()), asr: Mutex::new(None), asr_label: Mutex::new(None), + omni_pcm: Mutex::new(None), recorder: Mutex::new(None), audio_archive_active: AtomicBool::new(false), recording_mute: Mutex::new(SharedRecordingMuteState::new()), @@ -865,6 +863,7 @@ impl Coordinator { capsule_style: AtomicU8::new(0), capsule_cursor_passthrough: AtomicBool::new(true), qa_asr: Mutex::new(None), + qa_omni_pcm: Mutex::new(None), qa_recorder: Mutex::new(None), qa_stream_cancelled: Arc::new(AtomicBool::new(false)), local_asr_cache: Arc::new(crate::asr::local::LocalAsrCache::new()), @@ -945,6 +944,7 @@ impl Coordinator { state: Mutex::new(SessionState::default()), asr: Mutex::new(None), asr_label: Mutex::new(None), + omni_pcm: Mutex::new(None), recorder: Mutex::new(None), audio_archive_active: AtomicBool::new(false), recording_mute: Mutex::new(SharedRecordingMuteState::new()), @@ -983,6 +983,7 @@ impl Coordinator { capsule_style: AtomicU8::new(0), capsule_cursor_passthrough: AtomicBool::new(true), qa_asr: Mutex::new(None), + qa_omni_pcm: Mutex::new(None), qa_recorder: Mutex::new(None), qa_stream_cancelled: Arc::new(AtomicBool::new(false)), local_asr_cache: Arc::new(crate::asr::local::LocalAsrCache::new()), @@ -1061,7 +1062,6 @@ impl Coordinator { self.inner.local_asr_cache.loaded_model_id() } - /// 主动把当前本地 ASR 引擎状态推给前端（keepLoadedSecs 变更等命令侧调用）。 pub fn emit_local_asr_engine_status(&self) { emit_local_asr_engine_status(&self.inner); @@ -1559,7 +1559,6 @@ impl Coordinator { close_qa_panel(&self.inner); } - /// 用户点 ✕ / 按 Esc 关 Less Computer 浮窗：隐藏窗口 + 结束连续对话 /// （下次说话开新会话，不再 --continue 续旧上下文）。 pub fn less_computer_window_dismiss(&self) { @@ -1599,8 +1598,7 @@ impl Coordinator { // callback (SIGABRT). Tauri's runtime handle is safe from either thread. tauri::async_runtime::spawn(async move { let session_id = crate::coordinator_state::new_session_id(); - if let Err(e) = - dictation::run_voice_agent_transcript(&inner, session_id, text, 0).await + if let Err(e) = dictation::run_voice_agent_transcript(&inner, session_id, text, 0).await { log::warn!("[less-computer] text submit run failed: {e}"); } @@ -1624,10 +1622,7 @@ impl Coordinator { /// 执行——用户反馈「切换成默认风格后仍显示流光 Siri」。在保存路径直接同步后， /// 任何平台的下一次录音从入场帧起就携带最新样式，不再依赖 emit 闭包的时序。 pub fn sync_capsule_style_from_preferences(&self) { - let classic = matches!( - self.inner.prefs.get().capsule_style, - CapsuleStyle::Classic - ); + let classic = matches!(self.inner.prefs.get().capsule_style, CapsuleStyle::Classic); self.inner .capsule_style .store(if classic { 1 } else { 0 }, Ordering::Relaxed); @@ -2033,10 +2028,8 @@ impl Coordinator { .style_packs .get_or_default_active(&prefs.active_style_pack_id) .map_err(|e| e.to_string())?; - let style_system_prompt = crate::types::style_pack_prompt( - &pack, - crate::types::StylePromptKind::DictationAsr, - ); + let style_system_prompt = + crate::types::style_pack_prompt(&pack, crate::types::StylePromptKind::DictationAsr); let working_languages = prefs.working_languages; let chinese_script_preference = prefs.chinese_script_preference; let output_language_preference = prefs.output_language_preference; @@ -2079,6 +2072,7 @@ impl Coordinator { // repolish 不回写历史的模型/耗时字段，调用快照就地丢弃。 &mut None, &mut None, + pipeline_multimodal_enabled(&self.inner.prefs.get()), ) .await .map_err(|e| e.to_string()) @@ -2086,10 +2080,7 @@ impl Coordinator { /// 返回 (转写文本, 本次实际构建的 ASR (provider, model) 快照)。快照供命令层把 /// 「重转用了哪个模型」写回历史（构建时归因，PR #826 review）。 - pub async fn retranscribe_pcm( - &self, - pcm: Vec, - ) -> Result<(String, AsrCallLabel), String> { + pub async fn retranscribe_pcm(&self, pcm: Vec) -> Result<(String, AsrCallLabel), String> { self.retranscribe_pcm_inner(pcm, false, None).await } @@ -2183,22 +2174,18 @@ impl Coordinator { .map_err(|e| e.to_string())?, ActiveAsr::DashScopeMultimodal(m) => { tokio::time::timeout(m.transcribe_timeout(audio_secs), m.transcribe()) - .await - .map_err(|_| "重新转录超时".to_string())? - .map_err(|e| e.to_string())? - } - ActiveAsr::ElevenLabs(e) => { - tokio::time::timeout(elevenlabs_timeout, e.transcribe()) - .await - .map_err(|_| "重新转录超时".to_string())? - .map_err(|e| e.to_string())? - } - ActiveAsr::ElevenLabs(e) => { - tokio::time::timeout(elevenlabs_timeout, e.transcribe()) .await .map_err(|_| "重新转录超时".to_string())? .map_err(|e| e.to_string())? } + ActiveAsr::ElevenLabs(e) => tokio::time::timeout(elevenlabs_timeout, e.transcribe()) + .await + .map_err(|_| "重新转录超时".to_string())? + .map_err(|e| e.to_string())?, + ActiveAsr::ElevenLabs(e) => tokio::time::timeout(elevenlabs_timeout, e.transcribe()) + .await + .map_err(|_| "重新转录超时".to_string())? + .map_err(|e| e.to_string())?, #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => { let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; @@ -2462,9 +2449,11 @@ pub(super) fn insert_via_non_tsf_fallback( let prefs = inner.prefs.get(); let sendinput_options = dictation::windows_sendinput_options_from_prefs(&prefs); let status = finish_non_tsf_insertion_fallback( - || inner - .inserter - .insert_via_unicode_keystrokes(polished, sendinput_options), + || { + inner + .inserter + .insert_via_unicode_keystrokes(polished, sendinput_options) + }, || inner.inserter.copy_fallback(polished), ); @@ -2566,7 +2555,6 @@ mod non_tsf_fallback_tests { // ─────────────────────────── helpers ─────────────────────────── - fn read_whisper_credentials() -> (String, String, String) { let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) .ok() @@ -2784,10 +2772,12 @@ fn read_volc_credentials() -> VolcengineCredentials { // 密钥槽位随鉴权模式：AppIdToken 读旧版 Access Token，ApiKey 读独立的方舟 API Key， // 两者互不污染，切换模式不会把旧模式的凭据带进新模式的握手。 let secret = match auth_mode { - VolcengineAuthMode::AppIdToken => CredentialsVault::get(CredentialAccount::VolcengineAccessKey) - .ok() - .flatten() - .unwrap_or_default(), + VolcengineAuthMode::AppIdToken => { + CredentialsVault::get(CredentialAccount::VolcengineAccessKey) + .ok() + .flatten() + .unwrap_or_default() + } VolcengineAuthMode::ApiKey => CredentialsVault::get(CredentialAccount::VolcengineApiKey) .ok() .flatten() @@ -2831,7 +2821,6 @@ fn enabled_hotwords(inner: &Arc) -> Vec { .collect() } - /// 读 Gemini 凭据。所有 LLM provider 共用 ark.* 槽位（persistence 没做 per-provider /// 隔离），所以这里也是从 `ArkApiKey` / `ArkModelId` / `ArkEndpoint` 三个槽读， /// 但回退默认值改成谷歌的：base_url 默认 `https://generativelanguage.googleapis.com/v1beta`， @@ -2920,6 +2909,86 @@ fn build_active_llm_provider(llm_thinking_enabled: bool) -> anyhow::Result bool { + prefs.multimodal_pipeline_enabled + && prefs.pipeline_mode == crate::types::PipelineMode::Multimodal +} + +/// 多模态（Omni）模型通道的凭据预检（友好错误信息，供录音前拦截）。 +pub(crate) fn ensure_omni_credentials() -> Result<(), String> { + let api_key = CredentialsVault::get(CredentialAccount::OmniApiKey) + .map_err(|e| e.to_string())? + .unwrap_or_default(); + let model = CredentialsVault::get(CredentialAccount::OmniModel) + .map_err(|e| e.to_string())? + .unwrap_or_default(); + let base_url = CredentialsVault::get(CredentialAccount::OmniEndpoint) + .map_err(|e| e.to_string())? + .unwrap_or_default(); + if api_key.trim().is_empty() { + return Err("多模态模型 API Key 为空：请在 服务 → AI 提供商 → 多模态模型 中配置".into()); + } + if model.trim().is_empty() { + return Err("多模态模型 id 为空：请在 服务 → AI 提供商 → 多模态模型 中配置".into()); + } + let active = CredentialsVault::get_active_omni(); + if active != crate::omni::OMNI_GEMINI_PROVIDER_ID && base_url.trim().is_empty() { + return Err("多模态模型 Base URL 为空：请在 服务 → AI 提供商 → 多模态模型 中配置".into()); + } + Ok(()) +} + +fn omni_default_base_url(provider: &str) -> &'static str { + match provider { + "openai" => "https://api.openai.com/v1", + crate::omni::OMNI_GEMINI_PROVIDER_ID => "https://generativelanguage.googleapis.com/v1beta", + "dashscope-omni" => "https://dashscope.aliyuncs.com/compatible-mode/v1", + _ => "", + } +} + +/// 读取 omni 命名空间凭据并构建多模态模型通道（与 build_active_llm_provider +/// 平行的唯一构建点）。Gemini 按 provider id / base_url 路由到原生通道。 +pub(crate) fn build_active_omni_provider( + thinking_enabled: bool, +) -> anyhow::Result { + let active = CredentialsVault::get_active_omni(); + let api_key = CredentialsVault::get(CredentialAccount::OmniApiKey)?.unwrap_or_default(); + let model = CredentialsVault::get(CredentialAccount::OmniModel)?.unwrap_or_default(); + let base_url = CredentialsVault::get(CredentialAccount::OmniEndpoint)?.unwrap_or_default(); + if api_key.trim().is_empty() { + anyhow::bail!("多模态模型 API Key 为空"); + } + if model.trim().is_empty() { + anyhow::bail!("多模态模型 id 为空"); + } + let base_url = if base_url.trim().is_empty() { + omni_default_base_url(&active).to_string() + } else { + base_url.trim().to_string() + }; + if base_url.is_empty() { + anyhow::bail!("多模态模型 Base URL 为空"); + } + // 与 LLM / ASR 通道一致：拒绝指向内网/回环/元数据服务的地址（SSRF 防线）。 + crate::endpoint_security::validate_http_endpoint(&base_url) + .map_err(|_| anyhow::anyhow!("endpointInvalid"))?; + let config = crate::omni::OmniConfig { + provider_id: active.clone(), + base_url, + api_key, + model, + extra_headers: CredentialsVault::get_active_omni_extra_headers(), + temperature: crate::polish::openai_compatible_temperature_for_provider( + &active, + CredentialsVault::get_active_omni_temperature(), + ), + thinking_enabled, + }; + Ok(crate::omni::OmniProvider::new(config)) +} + fn resolve_ark_endpoint(api_key: &str) -> anyhow::Result { let endpoint = CredentialsVault::get(CredentialAccount::ArkEndpoint)?.filter(|s| !s.is_empty()); resolve_ark_endpoint_with_policy(api_key, endpoint) @@ -2957,8 +3026,16 @@ mod tests { // 非 volc. 命名空间 / 含异常字符 / 超长的值可能携带租户信息，一律不落历史。 assert_eq!(super::volc_resource_history_label(""), None); assert_eq!(super::volc_resource_history_label("my-secret-tenant"), None); - assert_eq!(super::volc_resource_history_label("volc.a b"), None, "空格不在字符集"); - assert_eq!(super::volc_resource_history_label("volc.引擎"), None, "非 ASCII 拒绝"); + assert_eq!( + super::volc_resource_history_label("volc.a b"), + None, + "空格不在字符集" + ); + assert_eq!( + super::volc_resource_history_label("volc.引擎"), + None, + "非 ASCII 拒绝" + ); let too_long = format!("volc.{}", "x".repeat(64)); assert_eq!(super::volc_resource_history_label(&too_long), None); } @@ -2975,6 +3052,24 @@ mod tests { Uuid::from_u128(n) } + #[test] + fn pipeline_multimodal_enabled_requires_both_flag_and_mode() { + let mut prefs = crate::types::UserPreferences::default(); + assert!(!super::pipeline_multimodal_enabled(&prefs)); + prefs.multimodal_pipeline_enabled = true; + assert!( + !super::pipeline_multimodal_enabled(&prefs), + "只开实验开关但模式还是 traditional 时不得启用" + ); + prefs.pipeline_mode = crate::types::PipelineMode::Multimodal; + assert!(super::pipeline_multimodal_enabled(&prefs)); + prefs.multimodal_pipeline_enabled = false; + assert!( + !super::pipeline_multimodal_enabled(&prefs), + "实验开关关闭时即使模式为 multimodal 也不得启用" + ); + } + #[test] fn failed_remote_pin_persistence_keeps_memory_and_server_state() { let slot = Mutex::new(Some("123456".to_string())); @@ -3247,7 +3342,9 @@ mod tests { fn openai_compatible_preset_is_whisper_compatible_and_conservative_by_default() { use crate::asr::whisper::AsrRequestFormat; - assert!(is_whisper_compatible_provider(OPENAI_COMPATIBLE_ASR_PROVIDER_ID)); + assert!(is_whisper_compatible_provider( + OPENAI_COMPATIBLE_ASR_PROVIDER_ID + )); assert_eq!( active_asr_provider_kind(OPENAI_COMPATIBLE_ASR_PROVIDER_ID), ActiveAsrProviderKind::WhisperCompatible @@ -3318,9 +3415,7 @@ mod tests { AdvancedAsrConfig::default() ); assert_eq!( - parse_advanced_asr_config(Some( - r#"{"verboseJson":false,"chunkDurationMs":30000}"# - )), + parse_advanced_asr_config(Some(r#"{"verboseJson":false,"chunkDurationMs":30000}"#)), AdvancedAsrConfig { verbose_json: false, chunk_duration_ms: Some(30_000), @@ -3494,8 +3589,8 @@ mod tests { // 穷尽 match，这里逐 kind 钉死映射，防止未来悄悄改动某个 provider 的凭据形态。 #[test] fn preflight_credential_maps_every_kind() { - use AsrPreflightCredential::*; use ActiveAsrProviderKind::*; + use AsrPreflightCredential::*; assert_eq!(Bailian.preflight_credential(), AsrApiKey); assert_eq!(Qwen3Realtime.preflight_credential(), AsrApiKey); assert_eq!(Mimo.preflight_credential(), AsrApiKey); @@ -3519,8 +3614,7 @@ mod tests { crate::asr::qwen_realtime::PROVIDER_ID ); assert_eq!( - resolve_effective_asr_provider(bailian, "qwen3-asr-flash-realtime-2026-02-10") - .unwrap(), + resolve_effective_asr_provider(bailian, "qwen3-asr-flash-realtime-2026-02-10").unwrap(), crate::asr::qwen_realtime::PROVIDER_ID ); assert_eq!( @@ -3586,22 +3680,20 @@ mod tests { .unwrap_err(); assert!(error.contains("不支持的百炼 ASR 模型")); // qwen3-asr-flash-filetrans 仅接受公网 URL，与本地录音链路不兼容，同样拒绝。 - let error = - resolve_effective_asr_provider(crate::asr::bailian::PROVIDER_ID, "qwen3-asr-flash-filetrans") - .unwrap_err(); + let error = resolve_effective_asr_provider( + crate::asr::bailian::PROVIDER_ID, + "qwen3-asr-flash-filetrans", + ) + .unwrap_err(); assert!(error.contains("不支持的百炼 ASR 模型")); } #[test] fn validates_only_supported_dashscope_multimodal_models() { assert!(validate_dashscope_multimodal_model("").is_ok()); - assert!( - validate_dashscope_multimodal_model("fun-asr-flash-2026-06-15").is_ok() - ); + assert!(validate_dashscope_multimodal_model("fun-asr-flash-2026-06-15").is_ok()); assert!(validate_dashscope_multimodal_model("qwen-audio-3.0-asr-flash").is_ok()); - assert!( - validate_dashscope_multimodal_model("qwen-audio-3.0-asr-flash-streaming").is_err() - ); + assert!(validate_dashscope_multimodal_model("qwen-audio-3.0-asr-flash-streaming").is_err()); } #[test] @@ -3636,8 +3728,8 @@ mod tests { #[test] fn configured_fields_maps_every_kind() { - use AsrConfiguredFields::*; use ActiveAsrProviderKind::*; + use AsrConfiguredFields::*; assert_eq!(Bailian.configured_fields(), ApiKeyOnly); assert_eq!(Qwen3Realtime.configured_fields(), ApiKeyOnly); assert_eq!(Mimo.configured_fields(), ApiKeyEndpointModel); @@ -3889,10 +3981,22 @@ mod tests { // 旧 schedule 触发时若期间有更新的 emit，应跳过隐藏（voice agent 取消双 emit 竞争）。 emit_capsule(&coordinator.inner, CapsuleState::Done, 0.0, 0, None, None); schedule_capsule_idle(&coordinator.inner, 30); - emit_capsule(&coordinator.inner, CapsuleState::Cancelled, 0.0, 0, None, None); + emit_capsule( + &coordinator.inner, + CapsuleState::Cancelled, + 0.0, + 0, + None, + None, + ); tokio::time::sleep(std::time::Duration::from_millis(120)).await; assert_eq!( - coordinator.inner.last_capsule_state.lock().as_ref().copied(), + coordinator + .inner + .last_capsule_state + .lock() + .as_ref() + .copied(), Some(CapsuleState::Cancelled), "旧 schedule 不应把更新的 Cancelled 状态提前隐藏" ); @@ -3905,7 +4009,12 @@ mod tests { schedule_capsule_idle(&coordinator.inner, 30); tokio::time::sleep(std::time::Duration::from_millis(120)).await; assert_eq!( - coordinator.inner.last_capsule_state.lock().as_ref().copied(), + coordinator + .inner + .last_capsule_state + .lock() + .as_ref() + .copied(), Some(CapsuleState::Idle), "无新 emit 时 schedule 应隐藏胶囊" ); @@ -4001,8 +4110,7 @@ mod tests { let coordinator = Coordinator::new(); // Idle + 冷却未过期：模拟「识别中按下 → 会话收尾 → bridge 取出该 Pressed」的时刻。 *coordinator.inner.session_cooldown_until.lock() = Some( - std::time::Instant::now() - + std::time::Duration::from_millis(POST_SESSION_COOLDOWN_MS), + std::time::Instant::now() + std::time::Duration::from_millis(POST_SESSION_COOLDOWN_MS), ); handle_pressed_edge(&coordinator.inner, std::time::Instant::now(), 1).await; @@ -4070,7 +4178,11 @@ mod tests { .hotkey_trigger_held .store(true, Ordering::SeqCst); - handle_released_edge(&coordinator.inner, pressed_at + std::time::Duration::from_millis(100)).await; + handle_released_edge( + &coordinator.inner, + pressed_at + std::time::Duration::from_millis(100), + ) + .await; // 短按松手不结束录音，等下一次按下再停。 assert_eq!( @@ -4099,7 +4211,10 @@ mod tests { ) .await; - assert_eq!(coordinator.inner.state.lock().phase, SessionPhase::Listening); + assert_eq!( + coordinator.inner.state.lock().phase, + SessionPhase::Listening + ); assert!(coordinator.inner.hotkey_press_at.lock().is_none()); } @@ -4117,7 +4232,11 @@ mod tests { .hotkey_trigger_held .store(true, Ordering::SeqCst); - handle_released_edge(&coordinator.inner, pressed_at + std::time::Duration::from_millis(500)).await; + handle_released_edge( + &coordinator.inner, + pressed_at + std::time::Duration::from_millis(500), + ) + .await; // 无 recorder / ASR 的测试会话下，end_session 直接收尾到 Idle。 assert_eq!(coordinator.inner.state.lock().phase, SessionPhase::Idle); diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index 672e5d834..b1f4ffd2a 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -283,6 +283,7 @@ async fn run_streaming_polish( prior_turns, llm_call, llm_elapsed_ms, + pipeline_multimodal_enabled(&inner.prefs.get()), ) .await; return (p, e, false); @@ -315,6 +316,7 @@ async fn run_streaming_polish( prior_turns, llm_call, llm_elapsed_ms, + pipeline_multimodal_enabled(&inner.prefs.get()), ) .await; return (p, err, false); @@ -327,8 +329,7 @@ async fn run_streaming_polish( // from what the user actually sees\"。 let (tx, rx) = std::sync::mpsc::channel::(); #[cfg(target_os = "windows")] - let sendinput_options = - windows_sendinput_options_from_prefs(&inner.prefs.get()); + let sendinput_options = windows_sendinput_options_from_prefs(&inner.prefs.get()); let typer_handle = tokio::task::spawn_blocking(move || { #[cfg(target_os = "windows")] { @@ -473,6 +474,7 @@ async fn run_streaming_polish( prior_turns, llm_call, llm_elapsed_ms, + pipeline_multimodal_enabled(&inner.prefs.get()), ) .await; (p, e, false) @@ -735,9 +737,7 @@ pub(super) async fn handle_pressed_edge( inner .hotkey_press_generation .store(press_id, Ordering::SeqCst); - inner - .hotkey_press_began_session - .store(0, Ordering::SeqCst); + inner.hotkey_press_began_session.store(0, Ordering::SeqCst); // 防抖：相邻 < HOTKEY_DEBOUNCE 的边沿直接丢弃，记到 log 方便排查。 // 与 `hotkey_trigger_held` 互补：held 防 press-without-release，本检查防 @@ -1052,8 +1052,13 @@ pub(super) async fn handle_released(inner: &Arc, released_at: std::time:: } if mode == HotkeyMode::Auto { // 使用物理按下/松开的事件时刻，避免 bridge 排队时把处理延迟误算为按住时长。 - let held_long = inner.hotkey_press_at.lock().take() - .map(|pressed_at| released_at.saturating_duration_since(pressed_at) >= AUTO_HOLD_THRESHOLD) + let held_long = inner + .hotkey_press_at + .lock() + .take() + .map(|pressed_at| { + released_at.saturating_duration_since(pressed_at) >= AUTO_HOLD_THRESHOLD + }) .unwrap_or(false); match phase { // 长按松手 = 按住说话，松手即停；短按 = 切换式，锁存保持录音，下次按下再停。 @@ -1065,9 +1070,7 @@ pub(super) async fn handle_released(inner: &Arc, released_at: std::time:: request_stop_during_starting(inner, "auto hold release edge"); } SessionPhase::Listening | SessionPhase::Starting => { - log::info!( - "[coord] auto short-tap latched (toggle semantics); next press stops" - ); + log::info!("[coord] auto short-tap latched (toggle semantics); next press stops"); } _ => {} } @@ -1371,8 +1374,7 @@ async fn run_less_computer_once( // OpenCode 无 `--settings`，护栏走 `permission` 配置经 OPENCODE_CONFIG_CONTENT 注入。 // build_opencode_guard_config 默认 bash deny 高风险前缀、webfetch deny，审批放行的 // 前缀显式 allow。fail-closed：序列化失败立即中止，绝不无护栏裸跑。 - let guard = - crate::coding_agent::guard::build_opencode_guard_config(&approved_patterns); + let guard = crate::coding_agent::guard::build_opencode_guard_config(&approved_patterns); let guard_str = match serde_json::to_string(&guard) { Ok(s) => s, Err(e) => { @@ -1586,10 +1588,7 @@ pub(super) async fn begin_session(inner: &Arc) -> Result<(), String> { /// begin_session 的带参版本，voice_agent=true 时在 Starting 阶段就标记好， /// 防止 finish_starting_session 处理 pending_stop 时丢失标志。 -pub(super) async fn begin_session_as( - inner: &Arc, - voice_agent: bool, -) -> Result<(), String> { +pub(super) async fn begin_session_as(inner: &Arc, voice_agent: bool) -> Result<(), String> { let current_session_id = { let mut state = inner.state.lock(); let Some(session_id) = @@ -1634,6 +1633,42 @@ pub(super) async fn begin_session_as( inner.capsule_warming.store(true, Ordering::SeqCst); emit_capsule(inner, CapsuleState::Recording, 0.0, 0, None, None); + // 多模态（Omni）模式：不构建 ASR，录音 PCM 直接进缓冲器，松键后一步出文。 + if pipeline_multimodal_enabled(&inner.prefs.get()) { + if let Err(message) = ensure_omni_credentials() { + log::warn!("[coord] omni credential gate failed: {message}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + 0, + Some(message.clone()), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + return Err(message); + } + if let Err(message) = ensure_microphone_permission(inner) { + log::warn!("[coord] omni microphone permission gate failed: {message}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + 0, + Some(message.clone()), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + return Err(message); + } + let consumer = PcmBufferConsumer::new(); + store_omni_pcm_for_session(inner, current_session_id, Arc::clone(&consumer)); + start_recorder_and_enter_listening(inner, current_session_id, "omni", consumer).await?; + return Ok(()); + } + if let Err(message) = ensure_asr_credentials() { log::warn!("[coord] ASR credential gate failed: {message}"); emit_capsule( @@ -2438,7 +2473,9 @@ pub(super) async fn start_recorder_for_starting( // 第一帧 PCM 真的流到 consumer 了（recorder.rs::process_callback 的顺序保证 // consume_pcm_chunk 先于 level_handler）——关掉预备态，让这一帧起 payload.warming // 翻 false，前端把「待命」光条点亮成正式录音态。之后每帧都是 false（幂等）。 - inner_for_level.capsule_warming.store(false, Ordering::SeqCst); + inner_for_level + .capsule_warming + .store(false, Ordering::SeqCst); emit_capsule( &inner_for_level, CapsuleState::Recording, @@ -2657,6 +2694,7 @@ fn build_transcribe_failed_session( asr_model: None, llm_provider: None, llm_model: None, + pipeline_mode: None, asr_ms: Some(asr_ms), polish_ms: None, } @@ -2924,6 +2962,90 @@ async fn wait_for_processing_cancel(inner: &Arc) { } } +/// 一次性（非流式）插入最终文本：平台分支与 `end_session` 原内联逻辑一致， +/// 供传统与多模态（Omni）两条收尾路径复用，避免插入策略漂移。 +async fn insert_final_text( + inner: &Arc, + current_session_id: SessionId, + text: &str, + prefs: &crate::types::UserPreferences, + focus_ready_for_paste: bool, +) -> InsertStatus { + let restore_clipboard = prefs.restore_clipboard_after_paste; + let allow_non_tsf_insertion_fallback = prefs.allow_non_tsf_insertion_fallback; + let windows_insertion_mode = prefs.windows_insertion_mode; + let paste_shortcut = prefs.paste_shortcut; + #[cfg(target_os = "android")] + { + crate::android::android_insert_with_strategy( + &inner.inserter, + text, + inner.prefs.get().android_insert_strategy, + ) + } + #[cfg(not(target_os = "android"))] + if focus_ready_for_paste { + #[cfg(target_os = "windows")] + { + match windows_insertion_mode { + crate::types::WindowsInsertionMode::SendInput => { + let sendinput_options = windows_sendinput_options_from_prefs(prefs); + if allow_non_tsf_insertion_fallback { + insert_via_non_tsf_fallback(inner, text, restore_clipboard, paste_shortcut) + } else { + inner + .inserter + .insert_via_unicode_keystrokes(text, sendinput_options) + } + } + crate::types::WindowsInsertionMode::Paste => { + inner + .inserter + .insert(text, restore_clipboard, paste_shortcut) + } + crate::types::WindowsInsertionMode::Tsf => { + let ime_target = capture_ime_submit_target(); + insert_with_windows_ime_first( + inner, + current_session_id, + text, + restore_clipboard, + allow_non_tsf_insertion_fallback, + paste_shortcut, + ime_target, + ) + .await + } + } + } + #[cfg(not(target_os = "windows"))] + { + inner + .inserter + .insert(text, restore_clipboard, paste_shortcut) + } + } else { + #[cfg(target_os = "linux")] + { + // Linux: fcitx5 commitString 无需窗口焦点，始终尝试插入。 + inner + .inserter + .insert(text, restore_clipboard, paste_shortcut) + } + #[cfg(not(target_os = "linux"))] + { + log::warn!( + "[coord] original insertion target is not foreground; copied output without paste" + ); + if allow_non_tsf_insertion_fallback { + inner.inserter.copy_fallback(text) + } else { + InsertStatus::Failed + } + } + } +} + pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { let current_session_id = { let mut state = inner.state.lock(); @@ -2941,6 +3063,12 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { release_recording_mute(inner, "dictation"); } + // 多模态（Omni）模式：不走 ASR 转写 + LLM 润色，录音 PCM 直接编码 WAV， + // 一次调用出最终文本（issue #902）。两套配置隔离，缺 omni 配置时明确报错。 + if pipeline_multimodal_enabled(&inner.prefs.get()) { + return finish_dictation_multimodal(inner, current_session_id, elapsed).await; + } + let asr_opt = take_asr_for_session(inner, current_session_id); // 构建时快照（begin_session 存入）。会话中途改设置不影响这份归因。 let mut asr_call_label = take_asr_label_for_session(inner, current_session_id); @@ -3429,9 +3557,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { // 处理最后一次重试结果时也复查一次取消标志，覆盖「重试刚返回 // Exhausted 与用户同时按 Esc」的窄竞态，避免误走失败提示。 if inner.state.lock().cancelled { - log::info!( - "[coord] cancel after silent ASR retry — discarding transcript" - ); + log::info!("[coord] cancel after silent ASR retry — discarding transcript"); restore_prepared_windows_ime_session(inner, current_session_id); finish_cancelled_processing(inner, current_session_id); return Ok(()); @@ -3498,6 +3624,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { asr_model: asr_model.clone(), llm_provider: None, llm_model: None, + pipeline_mode: None, asr_ms: Some(asr_ms), polish_ms: None, }; @@ -3597,10 +3724,8 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { let llm_thinking_enabled = prefs.llm_thinking_enabled; // 风格包原有 Prompt 就是录音 / ASR 后处理的完整规则；不要在全局设置再叠一层， // 否则会让同一个风格包的导出、复用和运行结果不一致。 - let style_system_prompt = crate::types::style_pack_prompt( - &pack, - crate::types::StylePromptKind::DictationAsr, - ); + let style_system_prompt = + crate::types::style_pack_prompt(&pack, crate::types::StylePromptKind::DictationAsr); let raw_uses_llm = mode == PolishMode::Raw && super::raw_style_pack_uses_llm(&pack); let translation_target = prefs.translation_target_language.trim().to_string(); let translation_active = @@ -3688,6 +3813,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { &prior_turns, &mut llm_call, &mut llm_elapsed_ms, + pipeline_multimodal_enabled(&inner.prefs.get()), ) .await; polish_source = src; @@ -3723,6 +3849,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { &prior_turns, &mut llm_call, &mut llm_elapsed_ms, + pipeline_multimodal_enabled(&inner.prefs.get()), ) .await; (p, e, false) @@ -3774,10 +3901,8 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { let focus_target = inner.state.lock().focus_target; let focus_ready_for_paste = restore_focus_target_if_possible(focus_target); let prefs = inner.prefs.get(); - let restore_clipboard = prefs.restore_clipboard_after_paste; let allow_non_tsf_insertion_fallback = prefs.allow_non_tsf_insertion_fallback; let windows_insertion_mode = prefs.windows_insertion_mode; - let paste_shortcut = prefs.paste_shortcut; // 流式路径下，字符已经通过 Unicode keystroke 落到光标处，跳过 inserter.insert。 let status = if already_streamed { log::info!( @@ -3787,80 +3912,14 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { ); InsertStatus::Inserted } else { - #[cfg(target_os = "android")] - { - crate::android::android_insert_with_strategy( - &inner.inserter, - &polished, - inner.prefs.get().android_insert_strategy, - ) - } - #[cfg(not(target_os = "android"))] - if focus_ready_for_paste { - #[cfg(target_os = "windows")] - { - match windows_insertion_mode { - crate::types::WindowsInsertionMode::SendInput => { - let sendinput_options = windows_sendinput_options_from_prefs(&prefs); - if allow_non_tsf_insertion_fallback { - insert_via_non_tsf_fallback( - inner, - &polished, - restore_clipboard, - paste_shortcut, - ) - } else { - inner - .inserter - .insert_via_unicode_keystrokes(&polished, sendinput_options) - } - } - crate::types::WindowsInsertionMode::Paste => inner.inserter.insert( - &polished, - restore_clipboard, - paste_shortcut, - ), - crate::types::WindowsInsertionMode::Tsf => { - let ime_target = capture_ime_submit_target(); - insert_with_windows_ime_first( - inner, - current_session_id, - &polished, - restore_clipboard, - allow_non_tsf_insertion_fallback, - paste_shortcut, - ime_target, - ) - .await - } - } - } - #[cfg(not(target_os = "windows"))] - { - inner - .inserter - .insert(&polished, restore_clipboard, paste_shortcut) - } - } else { - #[cfg(target_os = "linux")] - { - // Linux: fcitx5 commitString 无需窗口焦点，始终尝试插入。 - inner - .inserter - .insert(&polished, restore_clipboard, paste_shortcut) - } - #[cfg(not(target_os = "linux"))] - { - log::warn!( - "[coord] original insertion target is not foreground; copied output without paste" - ); - if allow_non_tsf_insertion_fallback { - inner.inserter.copy_fallback(&polished) - } else { - InsertStatus::Failed - } - } - } + insert_final_text( + inner, + current_session_id, + &polished, + &prefs, + focus_ready_for_paste, + ) + .await }; restore_prepared_windows_ime_session(inner, current_session_id); let inserted_chars = polished.chars().count() as u32; @@ -3924,6 +3983,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { asr_model, llm_provider, llm_model, + pipeline_mode: None, asr_ms: Some(asr_ms), polish_ms, }; @@ -3999,6 +4059,371 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { Ok(()) } +/// 多模态（Omni）听写收尾（issue #902）：录音 PCM → WAV → omni 一次调用 → +/// 修正规则 → 一次性插入 → 历史。与两段式管线完全隔离： +/// 不复用 ASR 构建/静默重试/流式插入，缺 omni 配置时明确报错、不回退传统配置。 +async fn finish_dictation_multimodal( + inner: &Arc, + current_session_id: SessionId, + elapsed: u64, +) -> Result<(), String> { + let Some(pcm_consumer) = take_omni_pcm_for_session(inner, current_session_id) else { + restore_prepared_windows_ime_session(inner, current_session_id); + if !finish_cancelled_processing(inner, current_session_id) { + set_phase_idle_if_session_matches(inner, current_session_id); + } + return Ok(()); + }; + let duration_ms = pcm_consumer.duration_ms(); + let wav = pcm_bytes_to_wav(&pcm_consumer.pcm()); + + // 录音后被取消 → 静默丢弃（与 ASR 完成后的 cancel 检查一致）。 + if inner.state.lock().cancelled { + log::info!("[coord] cancel detected after recording (multimodal) — discarding"); + restore_prepared_windows_ime_session(inner, current_session_id); + finish_cancelled_processing(inner, current_session_id); + return Ok(()); + } + + // 提示词装配：风格包提示词 + 词典热词 + 工作语言 + 翻译目标（同一次调用生效， + // 这正是多模态管线解决专有名词误识别的关键）；Less Computer 用逐字转写指令。 + let prefs = inner.prefs.get(); + let pack = match inner + .style_packs + .get_or_default_active(&prefs.active_style_pack_id) + { + Ok(pack) => pack, + Err(error) => { + log::warn!( + "[coord] active style pack unavailable, falling back to builtin light: {error}" + ); + crate::types::builtin_style_pack_for_mode(PolishMode::Light) + } + }; + let mode = pack.base_mode; + let translation_target = prefs.translation_target_language.trim().to_string(); + let translation_active = + inner.translation_modifier_seen.load(Ordering::SeqCst) && !translation_target.is_empty(); + let voice_agent = inner.state.lock().voice_agent; + + let system_prompt = if voice_agent { + "把用户的语音指令逐字转写为文本。不要改写、不要润色、不要补全，只输出转写文本本身。" + .to_string() + } else { + let base = + crate::types::style_pack_prompt(&pack, crate::types::StylePromptKind::DictationAsr); + let hotwords = enabled_phrases(inner); + let mut prompt = base; + if !prefs.working_languages.is_empty() { + prompt.push_str(&format!( + "\n\n# 工作语言\n用户主要在以下语言间工作：{}。", + prefs.working_languages.join("、") + )); + } + if !hotwords.is_empty() { + prompt.push_str(&format!( + "\n\n# 词典/热词\n以下专有名词必须严格按给定写法准确识别，不得换成同音错词：{}。", + hotwords.join("、") + )); + } + if translation_active { + prompt.push_str(&format!( + "\n\n用户按住了翻译键，需要把识别结果翻译成「{}」。直接输出译文，不要额外解释。", + translation_target + )); + } + prompt + }; + log::info!( + "[coord] multimodal dictation dispatch session_id={} mode={:?} translation={} voice_agent={} prompt_chars={} audio_ms={}", + current_session_id, + mode, + translation_active, + voice_agent, + system_prompt.chars().count(), + duration_ms + ); + + let provider = match build_active_omni_provider(prefs.llm_thinking_enabled) { + Ok(provider) => provider, + Err(error) => { + let reason = error.to_string(); + let user_msg = format!("多模态模型配置不完整：{reason}"); + return fail_dictation_multimodal(inner, current_session_id, elapsed, user_msg, reason); + } + }; + let omni_label = provider.call_label(); + let call_started = std::time::Instant::now(); + let output = match provider.complete(&system_prompt, "", Some(&wav)).await { + Ok(text) => text, + Err(error) => { + let reason = error.to_string(); + let user_msg = format!("多模态识别失败：{reason}"); + return fail_dictation_multimodal(inner, current_session_id, elapsed, user_msg, reason); + } + }; + let omni_ms = call_started.elapsed().as_millis() as u64; + let output = output.trim().to_string(); + + // 模型返回空 → emptyTranscript 失败历史 + 错误胶囊（保留录音供排查）。 + if output.is_empty() { + let session = DictationSession { + id: current_session_id.to_string(), + created_at: Utc::now().to_rfc3339(), + source: crate::types::HistorySource::Voice, + raw_transcript: String::new(), + final_text: String::new(), + mode: prefs.default_mode, + style_pack_id: None, + translation_active: false, + polish_source: None, + app_bundle_id: None, + app_name: None, + insert_status: InsertStatus::Failed, + error_code: Some("emptyTranscript".to_string()), + duration_ms: Some(duration_ms), + dictionary_entry_count: Some(enabled_phrases(inner).len() as u32), + has_audio_recording: Some(inner.audio_archive_active.load(Ordering::Relaxed)), + asr_provider: None, + asr_model: None, + llm_provider: Some(omni_label.provider.clone()), + llm_model: Some(omni_label.model.clone()), + pipeline_mode: Some("multimodal".to_string()), + asr_ms: None, + polish_ms: Some(omni_ms), + }; + let prefs_snapshot = inner.prefs.get(); + if let Err(e) = inner.history.append_with_retention( + session, + prefs_snapshot.history_retention_days, + prefs_snapshot.history_max_entries, + ) { + log::error!("[coord] history append failed: {e}"); + } + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some("多模态模型返回空结果".to_string()), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + { + let now = std::time::Instant::now(); + *inner.session_cooldown_until.lock() = + Some(now + std::time::Duration::from_millis(POST_SESSION_COOLDOWN_MS)); + } + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err("多模态模型返回空结果".to_string()); + } + + // Less Computer：转写文本交给 CLI agent，不走插入/历史（agent 流程自己收尾）。 + if voice_agent { + return run_voice_agent_transcript(inner, current_session_id, output, elapsed).await; + } + + let correction_rules = match inner.correction_rules.list() { + Ok(rules) => rules, + Err(e) => { + log::warn!("[coord] load correction rules failed: {e}; continue without correction"); + Vec::new() + } + }; + let polished = finalize_polished_text( + output, + translation_active, + false, + mode, + &None, + prefs.chinese_script_preference, + &correction_rules, + false, + ); + + // 原子化最后一次 cancel 检查 + 转 Inserting（与两段式路径同款 audit HIGH #2 修复）。 + let proceed_to_insert = { + let mut state = inner.state.lock(); + if state.cancelled { + false + } else { + state.phase = SessionPhase::Inserting; + true + } + }; + if !proceed_to_insert { + log::info!( + "[coord] cancel detected before insert (multimodal) — discarding output (chars={})", + polished.chars().count() + ); + restore_prepared_windows_ime_session(inner, current_session_id); + finish_cancelled_processing(inner, current_session_id); + return Ok(()); + } + + let focus_target = inner.state.lock().focus_target; + let focus_ready_for_paste = restore_focus_target_if_possible(focus_target); + let prefs = inner.prefs.get(); + let allow_non_tsf_insertion_fallback = prefs.allow_non_tsf_insertion_fallback; + let windows_insertion_mode = prefs.windows_insertion_mode; + let status = insert_final_text( + inner, + current_session_id, + &polished, + &prefs, + focus_ready_for_paste, + ) + .await; + restore_prepared_windows_ime_session(inner, current_session_id); + let inserted_chars = polished.chars().count() as u32; + + let total_hits: u64 = match inner.vocab.record_hits(&polished) { + Ok(n) => n, + Err(e) => { + log::error!("[coord] record_hits failed: {e}"); + 0 + } + }; + if total_hits > 0 { + if let Some(app) = inner.app.lock().clone() { + let _ = app.emit("vocab:updated", total_hits); + } + } + + let error_code = dictation_error_code( + status, + false, + focus_ready_for_paste, + allow_non_tsf_insertion_fallback, + windows_insertion_mode, + ) + .map(str::to_string); + let tsf_required_insert_failed = error_code.as_deref() == Some("windowsImeTsfRequired"); + + let prefs_snapshot = inner.prefs.get(); + let session = DictationSession { + id: current_session_id.to_string(), + created_at: Utc::now().to_rfc3339(), + source: crate::types::HistorySource::Voice, + raw_transcript: polished.clone(), + final_text: polished.clone(), + mode, + style_pack_id: Some(pack.id.clone()), + translation_active, + polish_source: None, + app_bundle_id: None, + app_name: None, + insert_status: status, + error_code, + duration_ms: Some(duration_ms), + dictionary_entry_count: Some(total_hits.min(u32::MAX as u64) as u32), + has_audio_recording: Some(inner.audio_archive_active.load(Ordering::Relaxed)), + asr_provider: None, + asr_model: None, + llm_provider: Some(omni_label.provider.clone()), + llm_model: Some(omni_label.model.clone()), + pipeline_mode: Some("multimodal".to_string()), + asr_ms: None, + polish_ms: Some(omni_ms), + }; + if let Err(e) = inner.history.append_with_retention( + session, + prefs_snapshot.history_retention_days, + prefs_snapshot.history_max_entries, + ) { + log::error!("[coord] history append failed: {e}"); + } + if let Err(e) = inner + .activity + .bump(&chrono::Local::now().format("%Y-%m-%d").to_string()) + { + log::warn!("[coord] activity bump failed: {e}"); + } + if !polished.trim().is_empty() { + if let Some(app) = inner.app.lock().clone() { + let _ = app.emit("remote:result", polished.clone()); + } + } + + let done_message = if tsf_required_insert_failed { + Some("TSF 未上屏，已禁止非 TSF 兜底".to_string()) + } else { + default_done_message(status, false) + }; + let session_failed = tsf_required_insert_failed || status == InsertStatus::Failed; + let capsule_state = if session_failed { + CapsuleState::Error + } else { + CapsuleState::Done + }; + emit_capsule( + inner, + capsule_state, + 0.0, + elapsed, + done_message, + Some(inserted_chars), + ); + + { + let mut state = inner.state.lock(); + state.phase = SessionPhase::Idle; + state.focus_target = None; + } + { + let now = std::time::Instant::now(); + *inner.session_cooldown_until.lock() = + Some(now + std::time::Duration::from_millis(POST_SESSION_COOLDOWN_MS)); + } + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + Ok(()) +} + +/// 多模态听写失败收尾：落失败历史（pipeline_mode=multimodal，前端据此隐藏 +/// 「重新转录」）→ 错误胶囊 → 恢复窗口/IME → 回 Idle + 冷却。永远返回 Err。 +fn fail_dictation_multimodal( + inner: &Arc, + session_id: SessionId, + elapsed: u64, + user_msg: String, + err: String, +) -> Result<(), String> { + let prefs = inner.prefs.get(); + let mut session = build_transcribe_failed_session( + session_id, + elapsed, + 0, + prefs.default_mode, + inner.audio_archive_active.load(Ordering::Relaxed), + ); + session.pipeline_mode = Some("multimodal".to_string()); + if let Err(e) = inner.history.append_with_retention( + session, + prefs.history_retention_days, + prefs.history_max_entries, + ) { + log::error!("[coord] transcribeFailed history append failed: {e}"); + } + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some(user_msg), + None, + ); + restore_prepared_windows_ime_session(inner, session_id); + inner.state.lock().phase = SessionPhase::Idle; + { + let now = std::time::Instant::now(); + *inner.session_cooldown_until.lock() = + Some(now + std::time::Duration::from_millis(POST_SESSION_COOLDOWN_MS)); + } + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + Err(err) +} + pub(super) fn dictation_error_code( status: InsertStatus, polish_failed: bool, @@ -4304,6 +4729,7 @@ mod tests { asr_model: None, llm_provider: None, llm_model: None, + pipeline_mode: None, asr_ms: None, polish_ms: None, } diff --git a/openless-all/app/src-tauri/src/coordinator/polish_flow.rs b/openless-all/app/src-tauri/src/coordinator/polish_flow.rs index 5bb8b765f..bf61c39c1 100644 --- a/openless-all/app/src-tauri/src/coordinator/polish_flow.rs +++ b/openless-all/app/src-tauri/src/coordinator/polish_flow.rs @@ -1,460 +1,494 @@ -//! Polish / translate orchestration extracted from `coordinator.rs` -//! (behavior-preserving move). -//! -//! The streaming/one-shot polish entry points and the polish+translate combiner. -//! References parent items via `use super::*;`; `pub(super)` so the parent and -//! sibling submodules (e.g. `dictation`) reach them through `use polish_flow::*;`. - -use super::*; - -/// 润色文本；失败时返回原文 + 失败原因，调用方据此弹错误胶囊 + 写历史 error_code。 -/// 之前固定返回 String，调用方拿不到失败信号 → 用户感知"为什么风格设置没生效"。issue #57。 -/// 流式润色的三态结果。让上层（dictation pipeline）能区分「已经流出去了」、 -/// 「降级到一次性」和「真失败了走 raw 兜底」三种 case。 -pub enum StreamingPolishOutcome { - /// 流式润色成功，`String` 是已经一边流一边交给 `on_delta` 的全部文本（用于写 - /// history、做词条命中统计）。调用方不应再 `inserter.insert(&text)`，因为字符 - /// 已经通过键盘事件落到光标处。 - Streamed(String), - /// 当前配置不支持流式：用户没开 streaming_insert / Gemini provider / Codex - /// provider / Raw 模式 / 翻译模式 / 不是 macOS。调用方应回到现有的 - /// `polish_or_passthrough` 一次性路径，跟历史行为完全一致。 - UnsupportedFallback, - /// 流式过程中失败（HTTP / 解析 / 空流等）。`String` 是失败原因，调用方应当 - /// 走 raw 兜底（同 `polish_or_passthrough` 失败分支的语义）。 - Failed(String), -} - -fn accumulate_llm_elapsed(total_ms: &mut Option, elapsed_ms: u64) { - *total_ms = Some(total_ms.unwrap_or(0).saturating_add(elapsed_ms)); -} - -fn record_llm_elapsed(total_ms: &mut Option, started: std::time::Instant) { - accumulate_llm_elapsed(total_ms, started.elapsed().as_millis() as u64); -} - -/// 流式润色入口。在不支持流式的所有 case 都返回 `UnsupportedFallback`，让调用方 -/// 透明降级。不修改任何持久化 / 焦点 / 光标状态。 -/// -/// `on_delta` 每收到一个 SSE chunk 就被调用一次（同步），调用方负责把 chunk 实际 -/// 模拟键盘事件落到光标 —— 见 `coordinator/dictation.rs` 的流式分支。 -/// `should_cancel` 用户取消时返回 true，立即 break SSE 读循环避免烧 quota。 -pub async fn polish_or_passthrough_streaming( - raw: &RawTranscript, - mode: PolishMode, - hotwords: &[String], - style_system_prompt: &str, - working_languages: &[String], - chinese_script_preference: ChineseScriptPreference, - output_language_preference: OutputLanguagePreference, - llm_thinking_enabled: bool, - front_app: Option<&str>, - prior_turns: &[(String, String)], - llm_call: &mut Option, - llm_elapsed_ms: &mut Option, - on_delta: F, - should_cancel: C, -) -> StreamingPolishOutcome -where - F: Fn(&str) + Send + Sync, - C: Fn() -> bool + Send + Sync, -{ - if mode == PolishMode::Raw && !raw_mode_uses_llm(style_system_prompt) { - log::info!("[coord] streaming polish skipped: mode=Raw, fall back to one-shot"); - return StreamingPolishOutcome::UnsupportedFallback; - } - let active_llm = CredentialsVault::get_active_llm(); - if active_llm == "gemini" { - log::info!( - "[coord] streaming polish skipped: active LLM provider=gemini (v1 not implemented), fall back to one-shot" - ); - return StreamingPolishOutcome::UnsupportedFallback; - } - let provider = match build_active_llm_provider(llm_thinking_enabled) { - Ok(p) => p, - Err(e) => { - log::error!("[coord] streaming polish: build provider failed: {e}"); - return StreamingPolishOutcome::Failed(e.to_string()); - } - }; - if !provider.supports_streaming_polish() { - log::info!( - "[coord] streaming polish skipped: provider does not support streaming (likely codex OAuth), fall back to one-shot" - ); - return StreamingPolishOutcome::UnsupportedFallback; - } - // 过了所有 early-out、即将发起真实调用——此刻才记录调用快照。 - *llm_call = Some(provider.call_label()); - log::info!( - "[coord] streaming polish START: provider=openai-compatible mode={:?} raw_chars={} prior_turns={}", - mode, - raw.text.chars().count(), - prior_turns.len() - ); - let call_started = std::time::Instant::now(); - let result = provider - .polish_streaming( - &raw.text, - mode, - hotwords, - style_system_prompt, - working_languages, - chinese_script_preference, - output_language_preference, - front_app, - prior_turns, - on_delta, - should_cancel, - ) - .await; - record_llm_elapsed(llm_elapsed_ms, call_started); - match result { - Ok(text) => { - log::info!( - "[coord] streaming polish OK: final_chars={}", - text.chars().count() - ); - StreamingPolishOutcome::Streamed(text) - } - Err(e) => { - let reason = e.to_string(); - log::error!("[coord] streaming polish FAILED: {reason}"); - StreamingPolishOutcome::Failed(reason) - } - } -} - +//! Polish / translate orchestration extracted from `coordinator.rs` +//! (behavior-preserving move). +//! +//! The streaming/one-shot polish entry points and the polish+translate combiner. +//! References parent items via `use super::*;`; `pub(super)` so the parent and +//! sibling submodules (e.g. `dictation`) reach them through `use polish_flow::*;`. + +use super::*; + +/// 润色文本；失败时返回原文 + 失败原因，调用方据此弹错误胶囊 + 写历史 error_code。 +/// 之前固定返回 String，调用方拿不到失败信号 → 用户感知"为什么风格设置没生效"。issue #57。 +/// 流式润色的三态结果。让上层（dictation pipeline）能区分「已经流出去了」、 +/// 「降级到一次性」和「真失败了走 raw 兜底」三种 case。 +pub enum StreamingPolishOutcome { + /// 流式润色成功，`String` 是已经一边流一边交给 `on_delta` 的全部文本（用于写 + /// history、做词条命中统计）。调用方不应再 `inserter.insert(&text)`，因为字符 + /// 已经通过键盘事件落到光标处。 + Streamed(String), + /// 当前配置不支持流式：用户没开 streaming_insert / Gemini provider / Codex + /// provider / Raw 模式 / 翻译模式 / 不是 macOS。调用方应回到现有的 + /// `polish_or_passthrough` 一次性路径，跟历史行为完全一致。 + UnsupportedFallback, + /// 流式过程中失败（HTTP / 解析 / 空流等）。`String` 是失败原因，调用方应当 + /// 走 raw 兜底（同 `polish_or_passthrough` 失败分支的语义）。 + Failed(String), +} + +fn accumulate_llm_elapsed(total_ms: &mut Option, elapsed_ms: u64) { + *total_ms = Some(total_ms.unwrap_or(0).saturating_add(elapsed_ms)); +} + +fn record_llm_elapsed(total_ms: &mut Option, started: std::time::Instant) { + accumulate_llm_elapsed(total_ms, started.elapsed().as_millis() as u64); +} + +/// 流式润色入口。在不支持流式的所有 case 都返回 `UnsupportedFallback`，让调用方 +/// 透明降级。不修改任何持久化 / 焦点 / 光标状态。 +/// +/// `on_delta` 每收到一个 SSE chunk 就被调用一次（同步），调用方负责把 chunk 实际 +/// 模拟键盘事件落到光标 —— 见 `coordinator/dictation.rs` 的流式分支。 +/// `should_cancel` 用户取消时返回 true，立即 break SSE 读循环避免烧 quota。 +pub async fn polish_or_passthrough_streaming( + raw: &RawTranscript, + mode: PolishMode, + hotwords: &[String], + style_system_prompt: &str, + working_languages: &[String], + chinese_script_preference: ChineseScriptPreference, + output_language_preference: OutputLanguagePreference, + llm_thinking_enabled: bool, + front_app: Option<&str>, + prior_turns: &[(String, String)], + llm_call: &mut Option, + llm_elapsed_ms: &mut Option, + on_delta: F, + should_cancel: C, +) -> StreamingPolishOutcome +where + F: Fn(&str) + Send + Sync, + C: Fn() -> bool + Send + Sync, +{ + if mode == PolishMode::Raw && !raw_mode_uses_llm(style_system_prompt) { + log::info!("[coord] streaming polish skipped: mode=Raw, fall back to one-shot"); + return StreamingPolishOutcome::UnsupportedFallback; + } + let active_llm = CredentialsVault::get_active_llm(); + if active_llm == "gemini" { + log::info!( + "[coord] streaming polish skipped: active LLM provider=gemini (v1 not implemented), fall back to one-shot" + ); + return StreamingPolishOutcome::UnsupportedFallback; + } + let provider = match build_active_llm_provider(llm_thinking_enabled) { + Ok(p) => p, + Err(e) => { + log::error!("[coord] streaming polish: build provider failed: {e}"); + return StreamingPolishOutcome::Failed(e.to_string()); + } + }; + if !provider.supports_streaming_polish() { + log::info!( + "[coord] streaming polish skipped: provider does not support streaming (likely codex OAuth), fall back to one-shot" + ); + return StreamingPolishOutcome::UnsupportedFallback; + } + // 过了所有 early-out、即将发起真实调用——此刻才记录调用快照。 + *llm_call = Some(provider.call_label()); + log::info!( + "[coord] streaming polish START: provider=openai-compatible mode={:?} raw_chars={} prior_turns={}", + mode, + raw.text.chars().count(), + prior_turns.len() + ); + let call_started = std::time::Instant::now(); + let result = provider + .polish_streaming( + &raw.text, + mode, + hotwords, + style_system_prompt, + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + prior_turns, + on_delta, + should_cancel, + ) + .await; + record_llm_elapsed(llm_elapsed_ms, call_started); + match result { + Ok(text) => { + log::info!( + "[coord] streaming polish OK: final_chars={}", + text.chars().count() + ); + StreamingPolishOutcome::Streamed(text) + } + Err(e) => { + let reason = e.to_string(); + log::error!("[coord] streaming polish FAILED: {reason}"); + StreamingPolishOutcome::Failed(reason) + } + } +} + pub(super) async fn polish_or_passthrough( raw: &RawTranscript, mode: PolishMode, hotwords: &[String], - style_system_prompt: &str, - working_languages: &[String], - chinese_script_preference: ChineseScriptPreference, - output_language_preference: OutputLanguagePreference, - llm_thinking_enabled: bool, - front_app: Option<&str>, - prior_turns: &[(String, String)], + style_system_prompt: &str, + working_languages: &[String], + chinese_script_preference: ChineseScriptPreference, + output_language_preference: OutputLanguagePreference, + llm_thinking_enabled: bool, + front_app: Option<&str>, + prior_turns: &[(String, String)], llm_call: &mut Option, llm_elapsed_ms: &mut Option, + multimodal: bool, ) -> (String, Option) { - if mode == PolishMode::Raw && !raw_mode_uses_llm(style_system_prompt) { - return (raw.text.clone(), None); - } - match polish_text( - &raw.text, - mode, - hotwords, - style_system_prompt, - working_languages, - chinese_script_preference, - output_language_preference, - llm_thinking_enabled, - front_app, - prior_turns, + if mode == PolishMode::Raw && !raw_mode_uses_llm(style_system_prompt) { + return (raw.text.clone(), None); + } + match polish_text( + &raw.text, + mode, + hotwords, + style_system_prompt, + working_languages, + chinese_script_preference, + output_language_preference, + llm_thinking_enabled, + front_app, + prior_turns, llm_call, llm_elapsed_ms, + multimodal, ) - .await - { - Ok(s) => (s, None), - Err(e) => { - let reason = e.to_string(); - log::error!("[coord] polish failed, falling back to raw: {reason}"); - (raw.text.clone(), Some(reason)) - } - } -} - + .await + { + Ok(s) => (s, None), + Err(e) => { + let reason = e.to_string(); + log::error!("[coord] polish failed, falling back to raw: {reason}"); + (raw.text.clone(), Some(reason)) + } + } +} + pub(super) async fn polish_text( raw: &str, mode: PolishMode, hotwords: &[String], style_system_prompt: &str, - working_languages: &[String], - chinese_script_preference: ChineseScriptPreference, - output_language_preference: OutputLanguagePreference, - llm_thinking_enabled: bool, - front_app: Option<&str>, - prior_turns: &[(String, String)], - llm_call: &mut Option, - llm_elapsed_ms: &mut Option, -) -> anyhow::Result { - // 谷歌 Gemini 分支：所有 LLM provider 共用 ark.* 凭据槽，唯独 Gemini 走原生 - // generateContent / 自带 thinkingConfig 控制；其余 provider 走 OpenAI - // 兼容协议，并在该路径里按 provider/channel 下发对应的思考开关。 - let active_llm = CredentialsVault::get_active_llm(); - if active_llm == "gemini" { - let (api_key, model, base_url) = read_gemini_credentials()?; - // 凭据读取成功、即将发起调用——记录构建时快照（preflight 失败走上面的 ? 提前返回，不会记）。 - *llm_call = Some(crate::polish::LlmCallLabel { - provider: active_llm.clone(), - model: model.clone(), - }); - let provider = GeminiProvider::new( - GeminiConfig::new(api_key, model, base_url).with_thinking_enabled(llm_thinking_enabled), - ); - let call_started = std::time::Instant::now(); - let result = provider - .polish( - raw, - mode, - hotwords, - style_system_prompt, - working_languages, - chinese_script_preference, - output_language_preference, - front_app, - prior_turns, - ) - .await; - record_llm_elapsed(llm_elapsed_ms, call_started); - return Ok(result?); - } - - let provider = build_active_llm_provider(llm_thinking_enabled)?; - *llm_call = Some(provider.call_label()); - let call_started = std::time::Instant::now(); - let result = provider - .polish( - raw, - mode, - hotwords, - style_system_prompt, - working_languages, - chinese_script_preference, - output_language_preference, - front_app, - prior_turns, - ) - .await; - record_llm_elapsed(llm_elapsed_ms, call_started); - Ok(result?) -} - -/// 专用翻译（仅翻译、不润色、单轮）。现作为"润色+翻译"合成调用解析失败时的兜底—— -/// 模型没按两段格式输出时，退回这里拿一段干净译文，而不是把畸形输出当译文插入。 -pub(super) async fn translate_text( - raw: &str, - target_language: &str, - working_languages: &[String], - chinese_script_preference: ChineseScriptPreference, - output_language_preference: OutputLanguagePreference, - llm_thinking_enabled: bool, - front_app: Option<&str>, + working_languages: &[String], + chinese_script_preference: ChineseScriptPreference, + output_language_preference: OutputLanguagePreference, + llm_thinking_enabled: bool, + front_app: Option<&str>, + prior_turns: &[(String, String)], llm_call: &mut Option, llm_elapsed_ms: &mut Option, + multimodal: bool, ) -> anyhow::Result { - // 见 polish_text 顶部注释——同样的 Gemini / OpenAI-compatible 路由逻辑。 - let active_llm = CredentialsVault::get_active_llm(); - if active_llm == "gemini" { - let (api_key, model, base_url) = read_gemini_credentials()?; + // 多模态（Omni）模式：纯文本管线（选区润色 / 历史重润色）改用 omni 模型当 + // 文本 LLM，读取 omni 命名空间凭据，与传统 LLM 配置隔离。 + if multimodal { + let provider = super::build_active_omni_provider(llm_thinking_enabled)?; + let label = provider.call_label(); *llm_call = Some(crate::polish::LlmCallLabel { - provider: active_llm.clone(), - model: model.clone(), + provider: label.provider, + model: label.model, }); - let provider = GeminiProvider::new( - GeminiConfig::new(api_key, model, base_url).with_thinking_enabled(llm_thinking_enabled), - ); + let mut system_prompt = style_system_prompt.to_string(); + if !hotwords.is_empty() { + system_prompt.push_str(&format!( + "\n\n# 词典/热词\n以下专有名词必须严格按给定写法准确识别：{}。", + hotwords.join("、") + )); + } + if !working_languages.is_empty() { + system_prompt.push_str(&format!( + "\n\n# 工作语言\n用户主要在以下语言间工作：{}。", + working_languages.join("、") + )); + } let call_started = std::time::Instant::now(); - let result = provider - .translate_to( - raw, - target_language, - working_languages, - chinese_script_preference, - output_language_preference, - front_app, - ) - .await; + let result = provider.complete(&system_prompt, raw, None).await; record_llm_elapsed(llm_elapsed_ms, call_started); return Ok(result?); } - let provider = build_active_llm_provider(llm_thinking_enabled)?; - *llm_call = Some(provider.call_label()); - let call_started = std::time::Instant::now(); - let result = provider - .translate_to( - raw, - target_language, - working_languages, - chinese_script_preference, - output_language_preference, - front_app, - ) - .await; - record_llm_elapsed(llm_elapsed_ms, call_started); - Ok(result?) -} - -/// "润色+翻译"单次调用的两段哨兵。模型按 `SRC\n源文\nTGT\n译文` 输出，解析器据此切分。 -/// 这两个串必须与 build_polish_translate_system_prompt 写给模型的完全一致。 -pub(super) const POLISH_TRANSLATE_SRC_MARKER: &str = "[[OPENLESS_POLISHED_SOURCE]]"; -pub(super) const POLISH_TRANSLATE_TGT_MARKER: &str = "[[OPENLESS_TRANSLATION]]"; - -/// 合成"先润色源文、再翻译"的系统提示词：在原翻译 prompt 之上追加"额外输出润色后源文" -/// 与严格两段格式（覆盖原 prompt 末尾的"只输出译文"）。译文仍是要插入用户光标的主产物， -/// 故完整保留原翻译规则；润色后的源文只作对话上下文用，轻量清理即可。 -pub(super) fn build_polish_translate_system_prompt(target_language: &str) -> String { - let base = crate::polish::prompts::translate_system_prompt(target_language); - format!( - "{base}\n\n\ - # 额外输出：润色后的源文（仅用于对话上下文，不展示给用户）\n\ - 在译文之前，先把上面的原始转写**按它本来的语言**润色一遍：去掉口癖（嗯 / 那个 / um）、\ - 补必要标点、纠正明显的识别错误，但**不翻译、不改写风格、不增删意思**。\n\n\ - # 输出格式（覆盖上面\u{201C}只输出译文\u{201D}的说明，严格遵守）\n\ - 严格按下面两段输出，两个标记必须原样出现、各占一行，标记之外不要有任何多余文字：\n\ - {src}\n\ - （这里放润色后的源文，保持原语言）\n\ - {tgt}\n\ - （这里放翻译成\u{300C}{lang}\u{300D}的译文）", - base = base, - src = POLISH_TRANSLATE_SRC_MARKER, - tgt = POLISH_TRANSLATE_TGT_MARKER, - lang = target_language, - ) -} - -/// 解析"润色+翻译"单次调用输出 → Some((润色后源文, 译文))。 -/// 找到译文标记且译文非空 → Some((源文, 译文))：源文标记缺失 / 源文段为空时源文为 None， -/// 译文取标记之后的干净正文。**没有译文标记、或译文段为空（模型截断 / 只吐了标记）→ None**， -/// 表示没拿到可信译文，交由调用方退回专用翻译——避免把空串当"成功译文"插进光标而丢字。 -pub(super) fn split_polish_translate_output(raw: &str) -> Option<(Option, String)> { - let tgt_idx = raw.find(POLISH_TRANSLATE_TGT_MARKER)?; - let translation = raw[tgt_idx + POLISH_TRANSLATE_TGT_MARKER.len()..] - .trim() - .to_string(); - if translation.is_empty() { - return None; - } - let before_tgt = &raw[..tgt_idx]; - let source = before_tgt - .find(POLISH_TRANSLATE_SRC_MARKER) - .map(|i| { - before_tgt[i + POLISH_TRANSLATE_SRC_MARKER.len()..] - .trim() - .to_string() - }) - .filter(|s| !s.is_empty()); - Some((source, translation)) -} - -/// 翻译路径——单次 LLM 调用同时润色源文 + 翻译。和 polish 一样失败时返回原文 + 失败原因， -/// 避免"不丢字"约定被违反（CLAUDE.md）。返回 (要插入的译文, 润色后源文供上下文用, 失败原因)。 -#[allow(clippy::too_many_arguments)] + // 谷歌 Gemini 分支：所有 LLM provider 共用 ark.* 凭据槽，唯独 Gemini 走原生 + // generateContent / 自带 thinkingConfig 控制；其余 provider 走 OpenAI + // 兼容协议，并在该路径里按 provider/channel 下发对应的思考开关。 + let active_llm = CredentialsVault::get_active_llm(); + if active_llm == "gemini" { + let (api_key, model, base_url) = read_gemini_credentials()?; + // 凭据读取成功、即将发起调用——记录构建时快照（preflight 失败走上面的 ? 提前返回，不会记）。 + *llm_call = Some(crate::polish::LlmCallLabel { + provider: active_llm.clone(), + model: model.clone(), + }); + let provider = GeminiProvider::new( + GeminiConfig::new(api_key, model, base_url).with_thinking_enabled(llm_thinking_enabled), + ); + let call_started = std::time::Instant::now(); + let result = provider + .polish( + raw, + mode, + hotwords, + style_system_prompt, + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + prior_turns, + ) + .await; + record_llm_elapsed(llm_elapsed_ms, call_started); + return Ok(result?); + } + + let provider = build_active_llm_provider(llm_thinking_enabled)?; + *llm_call = Some(provider.call_label()); + let call_started = std::time::Instant::now(); + let result = provider + .polish( + raw, + mode, + hotwords, + style_system_prompt, + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + prior_turns, + ) + .await; + record_llm_elapsed(llm_elapsed_ms, call_started); + Ok(result?) +} + +/// 专用翻译（仅翻译、不润色、单轮）。现作为"润色+翻译"合成调用解析失败时的兜底—— +/// 模型没按两段格式输出时，退回这里拿一段干净译文，而不是把畸形输出当译文插入。 +pub(super) async fn translate_text( + raw: &str, + target_language: &str, + working_languages: &[String], + chinese_script_preference: ChineseScriptPreference, + output_language_preference: OutputLanguagePreference, + llm_thinking_enabled: bool, + front_app: Option<&str>, + llm_call: &mut Option, + llm_elapsed_ms: &mut Option, +) -> anyhow::Result { + // 见 polish_text 顶部注释——同样的 Gemini / OpenAI-compatible 路由逻辑。 + let active_llm = CredentialsVault::get_active_llm(); + if active_llm == "gemini" { + let (api_key, model, base_url) = read_gemini_credentials()?; + *llm_call = Some(crate::polish::LlmCallLabel { + provider: active_llm.clone(), + model: model.clone(), + }); + let provider = GeminiProvider::new( + GeminiConfig::new(api_key, model, base_url).with_thinking_enabled(llm_thinking_enabled), + ); + let call_started = std::time::Instant::now(); + let result = provider + .translate_to( + raw, + target_language, + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + ) + .await; + record_llm_elapsed(llm_elapsed_ms, call_started); + return Ok(result?); + } + + let provider = build_active_llm_provider(llm_thinking_enabled)?; + *llm_call = Some(provider.call_label()); + let call_started = std::time::Instant::now(); + let result = provider + .translate_to( + raw, + target_language, + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + ) + .await; + record_llm_elapsed(llm_elapsed_ms, call_started); + Ok(result?) +} + +/// "润色+翻译"单次调用的两段哨兵。模型按 `SRC\n源文\nTGT\n译文` 输出，解析器据此切分。 +/// 这两个串必须与 build_polish_translate_system_prompt 写给模型的完全一致。 +pub(super) const POLISH_TRANSLATE_SRC_MARKER: &str = "[[OPENLESS_POLISHED_SOURCE]]"; +pub(super) const POLISH_TRANSLATE_TGT_MARKER: &str = "[[OPENLESS_TRANSLATION]]"; + +/// 合成"先润色源文、再翻译"的系统提示词：在原翻译 prompt 之上追加"额外输出润色后源文" +/// 与严格两段格式（覆盖原 prompt 末尾的"只输出译文"）。译文仍是要插入用户光标的主产物， +/// 故完整保留原翻译规则；润色后的源文只作对话上下文用，轻量清理即可。 +pub(super) fn build_polish_translate_system_prompt(target_language: &str) -> String { + let base = crate::polish::prompts::translate_system_prompt(target_language); + format!( + "{base}\n\n\ + # 额外输出：润色后的源文（仅用于对话上下文，不展示给用户）\n\ + 在译文之前，先把上面的原始转写**按它本来的语言**润色一遍：去掉口癖（嗯 / 那个 / um）、\ + 补必要标点、纠正明显的识别错误，但**不翻译、不改写风格、不增删意思**。\n\n\ + # 输出格式（覆盖上面\u{201C}只输出译文\u{201D}的说明，严格遵守）\n\ + 严格按下面两段输出，两个标记必须原样出现、各占一行，标记之外不要有任何多余文字：\n\ + {src}\n\ + （这里放润色后的源文，保持原语言）\n\ + {tgt}\n\ + （这里放翻译成\u{300C}{lang}\u{300D}的译文）", + base = base, + src = POLISH_TRANSLATE_SRC_MARKER, + tgt = POLISH_TRANSLATE_TGT_MARKER, + lang = target_language, + ) +} + +/// 解析"润色+翻译"单次调用输出 → Some((润色后源文, 译文))。 +/// 找到译文标记且译文非空 → Some((源文, 译文))：源文标记缺失 / 源文段为空时源文为 None， +/// 译文取标记之后的干净正文。**没有译文标记、或译文段为空（模型截断 / 只吐了标记）→ None**， +/// 表示没拿到可信译文，交由调用方退回专用翻译——避免把空串当"成功译文"插进光标而丢字。 +pub(super) fn split_polish_translate_output(raw: &str) -> Option<(Option, String)> { + let tgt_idx = raw.find(POLISH_TRANSLATE_TGT_MARKER)?; + let translation = raw[tgt_idx + POLISH_TRANSLATE_TGT_MARKER.len()..] + .trim() + .to_string(); + if translation.is_empty() { + return None; + } + let before_tgt = &raw[..tgt_idx]; + let source = before_tgt + .find(POLISH_TRANSLATE_SRC_MARKER) + .map(|i| { + before_tgt[i + POLISH_TRANSLATE_SRC_MARKER.len()..] + .trim() + .to_string() + }) + .filter(|s| !s.is_empty()); + Some((source, translation)) +} + +/// 翻译路径——单次 LLM 调用同时润色源文 + 翻译。和 polish 一样失败时返回原文 + 失败原因， +/// 避免"不丢字"约定被违反（CLAUDE.md）。返回 (要插入的译文, 润色后源文供上下文用, 失败原因)。 +#[allow(clippy::too_many_arguments)] pub(super) async fn polish_and_translate_or_passthrough( raw: &RawTranscript, target_language: &str, - mode: PolishMode, - hotwords: &[String], - working_languages: &[String], - chinese_script_preference: ChineseScriptPreference, - output_language_preference: OutputLanguagePreference, - llm_thinking_enabled: bool, - front_app: Option<&str>, - prior_turns: &[(String, String)], + mode: PolishMode, + hotwords: &[String], + working_languages: &[String], + chinese_script_preference: ChineseScriptPreference, + output_language_preference: OutputLanguagePreference, + llm_thinking_enabled: bool, + front_app: Option<&str>, + prior_turns: &[(String, String)], llm_call: &mut Option, llm_elapsed_ms: &mut Option, + multimodal: bool, ) -> (String, Option, Option) { - let system_prompt = build_polish_translate_system_prompt(target_language); - match polish_text( - &raw.text, - mode, - hotwords, - &system_prompt, - working_languages, - chinese_script_preference, - output_language_preference, - llm_thinking_enabled, - front_app, - prior_turns, + let system_prompt = build_polish_translate_system_prompt(target_language); + match polish_text( + &raw.text, + mode, + hotwords, + &system_prompt, + working_languages, + chinese_script_preference, + output_language_preference, + llm_thinking_enabled, + front_app, + prior_turns, llm_call, llm_elapsed_ms, + multimodal, ) - .await - { - Ok(out) => match split_polish_translate_output(&out) { - Some((source, translation)) => (translation, source, None), - None => { - // 模型没按两段格式输出：退回专用翻译拿一段干净译文，避免把畸形输出插进光标。 - // 此时无可信源文，这条翻译历史不参与后续普通润色上下文。 - log::warn!( - "[coord] polish+translate output missing markers; falling back to plain translate" - ); - match translate_text( - &raw.text, - target_language, - working_languages, - chinese_script_preference, - output_language_preference, - llm_thinking_enabled, - front_app, - llm_call, - llm_elapsed_ms, - ) - .await - { - Ok(translation) => (translation, None, None), - Err(e) => { - let reason = e.to_string(); - log::error!("[coord] fallback translate failed, using raw: {reason}"); - (raw.text.clone(), None, Some(reason)) - } - } - } - }, - Err(e) => { - let reason = e.to_string(); - log::error!("[coord] polish+translate failed, falling back to raw: {reason}"); - (raw.text.clone(), None, Some(reason)) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// PR #826 review：llm_call 快照只在真的构建 provider / 发起调用时填充。 - /// Raw 直通在读取任何凭据之前就 early-return，llm_call 必须保持 None—— - /// 调用方据此不落 llm_* / polish_ms。 - #[tokio::test] - async fn raw_passthrough_leaves_llm_call_snapshot_empty() { - let raw = RawTranscript { - text: "原样输出".to_string(), - duration_ms: 800, - }; - let mut llm_call: Option = None; - let mut llm_elapsed_ms = None; - // 直通判定：style prompt 等于内置 raw 提示词 → raw_mode_uses_llm 为 false。 - let builtin_raw_prompt = crate::types::StyleSystemPrompts::default().raw; - let (out, err) = polish_or_passthrough( - &raw, - PolishMode::Raw, - &[], - &builtin_raw_prompt, - &[], - ChineseScriptPreference::Auto, - OutputLanguagePreference::Auto, - false, - None, - &[], + .await + { + Ok(out) => match split_polish_translate_output(&out) { + Some((source, translation)) => (translation, source, None), + None => { + // 模型没按两段格式输出：退回专用翻译拿一段干净译文，避免把畸形输出插进光标。 + // 此时无可信源文，这条翻译历史不参与后续普通润色上下文。 + log::warn!( + "[coord] polish+translate output missing markers; falling back to plain translate" + ); + match translate_text( + &raw.text, + target_language, + working_languages, + chinese_script_preference, + output_language_preference, + llm_thinking_enabled, + front_app, + llm_call, + llm_elapsed_ms, + ) + .await + { + Ok(translation) => (translation, None, None), + Err(e) => { + let reason = e.to_string(); + log::error!("[coord] fallback translate failed, using raw: {reason}"); + (raw.text.clone(), None, Some(reason)) + } + } + } + }, + Err(e) => { + let reason = e.to_string(); + log::error!("[coord] polish+translate failed, falling back to raw: {reason}"); + (raw.text.clone(), None, Some(reason)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// PR #826 review：llm_call 快照只在真的构建 provider / 发起调用时填充。 + /// Raw 直通在读取任何凭据之前就 early-return，llm_call 必须保持 None—— + /// 调用方据此不落 llm_* / polish_ms。 + #[tokio::test] + async fn raw_passthrough_leaves_llm_call_snapshot_empty() { + let raw = RawTranscript { + text: "原样输出".to_string(), + duration_ms: 800, + }; + let mut llm_call: Option = None; + let mut llm_elapsed_ms = None; + // 直通判定：style prompt 等于内置 raw 提示词 → raw_mode_uses_llm 为 false。 + let builtin_raw_prompt = crate::types::StyleSystemPrompts::default().raw; + let (out, err) = polish_or_passthrough( + &raw, + PolishMode::Raw, + &[], + &builtin_raw_prompt, + &[], + ChineseScriptPreference::Auto, + OutputLanguagePreference::Auto, + false, + None, + &[], &mut llm_call, &mut llm_elapsed_ms, + false, ) .await; - assert_eq!(out, "原样输出"); - assert_eq!(err, None); - assert_eq!(llm_call, None, "Raw 直通不得产生 LLM 调用快照"); - assert_eq!(llm_elapsed_ms, None, "Raw 直通不得产生 LLM 调用耗时"); - } - - #[test] - fn llm_elapsed_accumulates_only_provider_call_durations() { - let mut elapsed_ms = None; - accumulate_llm_elapsed(&mut elapsed_ms, 120); - accumulate_llm_elapsed(&mut elapsed_ms, 80); - assert_eq!(elapsed_ms, Some(200)); - } -} + assert_eq!(out, "原样输出"); + assert_eq!(err, None); + assert_eq!(llm_call, None, "Raw 直通不得产生 LLM 调用快照"); + assert_eq!(llm_elapsed_ms, None, "Raw 直通不得产生 LLM 调用耗时"); + } + + #[test] + fn llm_elapsed_accumulates_only_provider_call_durations() { + let mut elapsed_ms = None; + accumulate_llm_elapsed(&mut elapsed_ms, 120); + accumulate_llm_elapsed(&mut elapsed_ms, 80); + assert_eq!(elapsed_ms, Some(200)); + } +} diff --git a/openless-all/app/src-tauri/src/coordinator/qa_session.rs b/openless-all/app/src-tauri/src/coordinator/qa_session.rs index 2a7566ef6..54506f041 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa_session.rs @@ -6,6 +6,7 @@ //! References parent items via `use super::*;`; `pub(super)` so the parent and //! sibling submodules (e.g. `qa`) reach them through `use qa_session::*;`. +use super::resources::*; use super::*; fn compose_qa_user_content(selection_text: &str, question: &str) -> String { @@ -206,6 +207,7 @@ pub(super) async fn finalize_dictation_as_qa_question(inner: &Arc) -> Res raw.text.trim().to_string(), raw.duration_ms, session_id, + None, ) .await } @@ -268,7 +270,7 @@ pub(super) async fn submit_qa_text_question( } } - answer_qa_question_text(inner, question, 0, session_id).await + answer_qa_question_text(inner, question, 0, session_id, None).await } pub(super) async fn take_current_dictation_transcript_for_qa( @@ -293,6 +295,26 @@ pub(super) async fn take_current_dictation_transcript_for_qa( release_recording_mute(inner, "dictation"); } + // 多模态（Omni）模式：dictation 会话没有 ASR，录音 PCM 直接交给 QA 一步回答。 + if pipeline_multimodal_enabled(&inner.prefs.get()) { + let Some(pcm_consumer) = take_omni_pcm_for_session(inner, current_session_id) else { + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + return Ok(None); + }; + let duration_ms = pcm_consumer.duration_ms(); + let wav = pcm_bytes_to_wav(&pcm_consumer.pcm()); + restore_prepared_windows_ime_session(inner, current_session_id); + { + let mut state = inner.state.lock(); + state.phase = SessionPhase::Idle; + state.focus_target = None; + } + answer_qa_question_text(inner, String::new(), duration_ms, qa_session_id, Some(wav)) + .await?; + return Ok(None); + } + let Some(asr) = take_asr_for_session(inner, current_session_id) else { restore_prepared_windows_ime_session(inner, current_session_id); set_phase_idle_if_session_matches(inner, current_session_id); @@ -605,6 +627,7 @@ pub(super) async fn answer_qa_question_text( question: String, duration_ms: u64, session_id: SessionId, + audio_wav: Option>, ) -> Result<(), String> { { let state = inner.qa_state.lock(); @@ -613,20 +636,27 @@ pub(super) async fn answer_qa_question_text( return Ok(()); } } - if question.trim().is_empty() { + if question.trim().is_empty() && audio_wav.is_none() { if qa_turn_can_continue(&inner.qa_state.lock(), session_id) { finish_qa_idle_silently_if_current(inner, session_id); } return Ok(()); } + // 多模态（Omni）模式：问题本体在音频里，文本槽位用占位符，便于模型理解 + // 「这是语音提问」并让 history 的 raw_transcript 不为空。 + let question_for_message = if audio_wav.is_some() { + "（语音问题）".to_string() + } else { + question.clone() + }; { let mut state = inner.qa_state.lock(); if !qa_turn_can_continue(&state, session_id) { log::info!("[coord] QA turn invalidated before answer dispatch"); return Ok(()); } - let user_message = qa_user_message_from_state(&state, &question); + let user_message = qa_user_message_from_state(&state, &question_for_message); state.messages.push(user_message); } @@ -702,6 +732,8 @@ pub(super) async fn answer_qa_question_text( output_language_preference, llm_thinking_enabled, front_app.as_deref(), + audio_wav, + pipeline_multimodal_enabled(&inner.prefs.get()), on_delta, should_cancel, ) @@ -771,6 +803,7 @@ pub(super) async fn answer_qa_question_text( asr_model: None, llm_provider: None, llm_model: None, + pipeline_mode: None, asr_ms: None, polish_ms: None, }; @@ -842,41 +875,65 @@ pub(super) async fn begin_qa_session(inner: &Arc) -> Result<(), String> { // 2. QA 与 dictation 使用同一个 active ASR 入口。不要回退火山，否则用户配置 // 百炼 / Whisper / 本地 ASR 后，浮窗仍会偷偷走另一套凭据。 - let active_asr = CredentialsVault::get_active_asr(); - if let Err(message) = ensure_asr_credentials() { - log::warn!("[coord] QA: active ASR credentials missing: {message}"); - finish_qa_with_error_if_current(inner, session_id, format!("缺少 ASR 凭据：{message}")); - return Err(message); - } - - if let Err(message) = ensure_microphone_permission(inner) { - log::warn!("[coord] QA: microphone permission gate failed: {message}"); - finish_qa_with_error_if_current(inner, session_id, message.clone()); - return Err(message); - } - - // QA 历史暂不落模型归因字段，构建时快照就地丢弃（dictation / 重转录路径在用）。 - let qa_asr = match build_qa_asr_start(inner, &active_asr).await { - Ok((qa_asr, _asr_call_label)) => qa_asr, - Err(message) => { - log::error!("[coord] QA active ASR init failed: {message}"); + // 多模态（Omni）模式：不构建 ASR，录音 PCM 进缓冲器，松键后一步出答案。 + let multimodal = pipeline_multimodal_enabled(&inner.prefs.get()); + let qa_asr: Option = if multimodal { + if let Err(message) = ensure_omni_credentials() { + log::warn!("[coord] QA: omni credential gate failed: {message}"); finish_qa_with_error_if_current( inner, session_id, - format!("ASR 初始化失败: {message}"), + format!("缺少多模态模型凭据：{message}"), ); return Err(message); } + None + } else { + let active_asr = CredentialsVault::get_active_asr(); + if let Err(message) = ensure_asr_credentials() { + log::warn!("[coord] QA: active ASR credentials missing: {message}"); + finish_qa_with_error_if_current(inner, session_id, format!("缺少 ASR 凭据：{message}")); + return Err(message); + } + // QA 历史暂不落模型归因字段，构建时快照就地丢弃（dictation / 重转录路径在用）。 + match build_qa_asr_start(inner, &active_asr).await { + Ok((qa_asr, _asr_call_label)) => Some(qa_asr), + Err(message) => { + log::error!("[coord] QA active ASR init failed: {message}"); + finish_qa_with_error_if_current( + inner, + session_id, + format!("ASR 初始化失败: {message}"), + ); + return Err(message); + } + } }; - let consumer = { + + if let Err(message) = ensure_microphone_permission(inner) { + log::warn!("[coord] QA: microphone permission gate failed: {message}"); + finish_qa_with_error_if_current(inner, session_id, message.clone()); + return Err(message); + } + + let consumer: Arc = { let state = inner.qa_state.lock(); if !qa_recording_can_continue(&state, session_id) { log::info!("[coord] QA recording invalidated during ASR initialization"); return Ok(()); } - let consumer = qa_asr.recorder_consumer(); - store_qa_asr_for_session(inner, session_id, qa_asr.active_asr()); - consumer + match &qa_asr { + Some(start) => { + let consumer = start.recorder_consumer(); + store_qa_asr_for_session(inner, session_id, start.active_asr()); + consumer + } + None => { + let consumer = PcmBufferConsumer::new(); + store_qa_omni_pcm_for_session(inner, session_id, Arc::clone(&consumer)); + consumer + } + } }; // QA recorder 不需要 RMS 节流到胶囊；前端 QA 浮窗有自己的电平视图， @@ -961,18 +1018,20 @@ pub(super) async fn begin_qa_session(inner: &Arc) -> Result<(), String> { } } - if let Err(e) = qa_asr.open_streaming_session().await { - if !qa_recording_can_continue(&inner.qa_state.lock(), session_id) { - log::info!("[coord] discarded ASR error from invalidated QA session"); + if let Some(start) = &qa_asr { + if let Err(e) = start.open_streaming_session().await { + if !qa_recording_can_continue(&inner.qa_state.lock(), session_id) { + log::info!("[coord] discarded ASR error from invalidated QA session"); + stop_qa_recorder_for_session(inner, session_id); + cancel_qa_asr_for_session(inner, session_id); + return Ok(()); + } + log::error!("[coord] QA: open ASR session failed: {e}"); stop_qa_recorder_for_session(inner, session_id); cancel_qa_asr_for_session(inner, session_id); - return Ok(()); + finish_qa_with_error_if_current(inner, session_id, format!("ASR 连接失败: {e}")); + return Err(e); } - log::error!("[coord] QA: open ASR session failed: {e}"); - stop_qa_recorder_for_session(inner, session_id); - cancel_qa_asr_for_session(inner, session_id); - finish_qa_with_error_if_current(inner, session_id, format!("ASR 连接失败: {e}")); - return Err(e); } // cancel race：在 await 期间用户可能 dismiss 了浮窗。 @@ -1017,6 +1076,18 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { stop_qa_recorder_for_session(inner, session_id); + // 多模态（Omni）模式：不走 ASR 转写，录音 PCM 直接编码 WAV，一步出答案。 + if pipeline_multimodal_enabled(&inner.prefs.get()) { + let Some(pcm_consumer) = take_qa_omni_pcm_for_session(inner, session_id) else { + reset_qa_processing_if_current(&mut inner.qa_state.lock(), session_id); + return Ok(()); + }; + let duration_ms = pcm_consumer.duration_ms(); + let wav = pcm_bytes_to_wav(&pcm_consumer.pcm()); + return answer_qa_question_text(inner, String::new(), duration_ms, session_id, Some(wav)) + .await; + } + let asr = match take_qa_asr_for_session(inner, session_id) { Some(a) => a, None => { @@ -1394,7 +1465,7 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { return Ok(()); } - answer_qa_question_text(inner, question, raw.duration_ms, session_id).await + answer_qa_question_text(inner, question, raw.duration_ms, session_id, None).await } /// 静默收尾：发 idle 事件给前端，phase 复位。**不关浮窗**（v2：浮窗只在用户 @@ -1471,6 +1542,8 @@ pub(super) async fn answer_chat_dispatch( output_language_preference: OutputLanguagePreference, llm_thinking_enabled: bool, front_app: Option<&str>, + audio_wav: Option>, + multimodal: bool, on_delta: F, should_cancel: C, ) -> anyhow::Result @@ -1478,6 +1551,50 @@ where F: Fn(&str) + Send + Sync, C: Fn() -> bool + Send + Sync, { + // 多模态（Omni）模式：音频 + 选区/历史上下文一次调用出答案。 + // OpenAI 兼容通道逐字流式（answer_delta）；Gemini 通道一次性返回。 + if let Some(wav) = audio_wav { + let provider = build_active_omni_provider(llm_thinking_enabled)?; + let system_prompt = crate::polish::compose_qa_system_prompt( + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + ); + let user_text = messages + .iter() + .map(|message| format!("{}: {}", message.role, message.content)) + .collect::>() + .join("\n\n"); + return Ok(provider + .complete_streaming( + &system_prompt, + &user_text, + Some(&wav), + on_delta, + should_cancel, + ) + .await?); + } + // 多模态模式下键盘输入的纯文本问题：omni 模型当文本 LLM 用（无音频 part）。 + if multimodal { + let provider = build_active_omni_provider(llm_thinking_enabled)?; + let system_prompt = crate::polish::compose_qa_system_prompt( + working_languages, + chinese_script_preference, + output_language_preference, + front_app, + ); + let user_text = messages + .iter() + .map(|message| format!("{}: {}", message.role, message.content)) + .collect::>() + .join("\n\n"); + return Ok(provider + .complete_streaming(&system_prompt, &user_text, None, on_delta, should_cancel) + .await?); + } + // 见 polish_text 顶部注释——同样的 Gemini / OpenAI-compatible 路由逻辑， // QA 流式回答走 Gemini 原生 :streamGenerateContent?alt=sse。 let active_llm = CredentialsVault::get_active_llm(); diff --git a/openless-all/app/src-tauri/src/coordinator/resources.rs b/openless-all/app/src-tauri/src/coordinator/resources.rs index 0ae3e0b0e..55c742c61 100644 --- a/openless-all/app/src-tauri/src/coordinator/resources.rs +++ b/openless-all/app/src-tauri/src/coordinator/resources.rs @@ -67,6 +67,74 @@ pub(super) fn store_asr_for_session( *inner.asr_label.lock() = Some(SessionResource::new(session_id, label)); } +/// 多模态模式下替代 ASR 消费录音 PCM 的简单缓冲器：录音期间把 16k/mono/i16 PCM +/// 原样攒进 Vec，松键后由 omni 通道编码成 WAV 一次调用。与 ActiveAsr 完全解耦， +/// 不会误触发任何 ASR 协议/凭据逻辑。 +#[derive(Default)] +pub(super) struct PcmBufferConsumer { + buffer: parking_lot::Mutex>, +} + +impl PcmBufferConsumer { + pub(super) fn new() -> Arc { + Arc::new(Self::default()) + } + + pub(super) fn pcm(&self) -> Vec { + self.buffer.lock().clone() + } + + pub(super) fn duration_ms(&self) -> u64 { + crate::asr::pcm::pcm_duration_ms(&self.buffer.lock()) + } +} + +impl crate::recorder::AudioConsumer for PcmBufferConsumer { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + self.buffer.lock().extend_from_slice(pcm); + } +} + +/// 把 16k/mono/i16 原始 PCM 字节编码成 WAV 文件字节（omni 通道统一入口）。 +/// 与各 ASR provider 内联的 `chunks_exact(2)` 转换等价，收口成共享实现。 +pub(super) fn pcm_bytes_to_wav(pcm: &[u8]) -> Vec { + let samples: Vec = pcm + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + crate::asr::wav::encode_wav_16k_mono(&samples) +} + +pub(super) fn store_omni_pcm_for_session( + inner: &Arc, + session_id: SessionId, + consumer: Arc, +) { + *inner.omni_pcm.lock() = Some(SessionResource::new(session_id, consumer)); +} + +pub(super) fn take_omni_pcm_for_session( + inner: &Arc, + session_id: SessionId, +) -> Option> { + take_session_resource(&mut inner.omni_pcm.lock(), session_id) +} + +pub(super) fn store_qa_omni_pcm_for_session( + inner: &Arc, + session_id: SessionId, + consumer: Arc, +) { + *inner.qa_omni_pcm.lock() = Some(SessionResource::new(session_id, consumer)); +} + +pub(super) fn take_qa_omni_pcm_for_session( + inner: &Arc, + session_id: SessionId, +) -> Option> { + take_session_resource(&mut inner.qa_omni_pcm.lock(), session_id) +} + pub(super) fn take_asr_for_session(inner: &Arc, session_id: SessionId) -> Option { let mut slot = inner.asr.lock(); take_session_resource(&mut slot, session_id) diff --git a/openless-all/app/src-tauri/src/coordinator/selection_polish.rs b/openless-all/app/src-tauri/src/coordinator/selection_polish.rs index 699f4020f..12894e184 100644 --- a/openless-all/app/src-tauri/src/coordinator/selection_polish.rs +++ b/openless-all/app/src-tauri/src/coordinator/selection_polish.rs @@ -10,8 +10,9 @@ use std::sync::{ }; use super::{ - emit_selection_polish_capsule, enabled_phrases, polish_text, raw_style_pack_uses_llm, - schedule_selection_polish_capsule_idle, Coordinator, Inner, CAPSULE_AUTO_HIDE_DELAY_MS, + emit_selection_polish_capsule, enabled_phrases, pipeline_multimodal_enabled, polish_text, + raw_style_pack_uses_llm, schedule_selection_polish_capsule_idle, Coordinator, Inner, + CAPSULE_AUTO_HIDE_DELAY_MS, }; use chrono::Utc; use serde::Serialize; @@ -198,10 +199,8 @@ pub(super) async fn run_selection_polish(inner: &Arc) -> Result<(), Strin // 与 `repolish` 同样读取当前 style pack、词表和语言偏好；但前台上下文必须 // 来自选区捕获时的源应用，避免在 provider 等待期间重新读取/校验目标窗口。 // 选区润色只读取风格包的书面文本 Prompt；旧包缺少该字段时回退为安全默认。 - let selection_style_prompt = crate::types::style_pack_prompt( - &pack, - crate::types::StylePromptKind::Selection, - ); + let selection_style_prompt = + crate::types::style_pack_prompt(&pack, crate::types::StylePromptKind::Selection); log::info!( "[style-pack] runtime dispatch scope=selection pack={} kind={:?} mode={:?} prompt_chars={}", pack.id, @@ -225,6 +224,7 @@ pub(super) async fn run_selection_polish(inner: &Arc) -> Result<(), Strin &[], &mut llm_call, &mut polish_ms, + pipeline_multimodal_enabled(&inner.prefs.get()), ) .await .map_err(|error| error.to_string()) @@ -296,7 +296,10 @@ pub(super) async fn run_selection_polish(inner: &Arc) -> Result<(), Strin finish_selection_polish_capsule( inner, CapsuleState::Done, - selection_polish_success_message(InsertStatus::Inserted, prefs.selection_polish_output_mode), + selection_polish_success_message( + InsertStatus::Inserted, + prefs.selection_polish_output_mode, + ), ); return Ok(()); } @@ -339,6 +342,7 @@ pub(super) async fn run_selection_polish(inner: &Arc) -> Result<(), Strin asr_model: None, llm_provider, llm_model, + pipeline_mode: None, asr_ms: None, polish_ms, }; @@ -459,6 +463,7 @@ impl Coordinator { asr_model: None, llm_provider: preview.llm_provider, llm_model: preview.llm_model, + pipeline_mode: None, asr_ms: None, polish_ms: preview.polish_ms, }; diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 39d1507a1..b15616fe9 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -49,6 +49,7 @@ mod llm_gemini; #[cfg(mobile)] mod mobile_runtime; mod net; +mod omni; mod permissions; mod persistence; mod polish; @@ -68,14 +69,14 @@ mod selection; mod selection; #[cfg(not(mobile))] mod shortcut_binding; +#[cfg(mobile)] +#[path = "mobile_stubs/shortcut_binding.rs"] +mod shortcut_binding; #[cfg(not(mobile))] mod side_aware_combo; #[cfg(mobile)] #[path = "mobile_stubs/side_aware_combo.rs"] mod side_aware_combo; -#[cfg(mobile)] -#[path = "mobile_stubs/shortcut_binding.rs"] -mod shortcut_binding; mod types; #[cfg(not(mobile))] mod unicode_keystroke; @@ -247,6 +248,7 @@ macro_rules! app_invoke_handler_desktop { commands::read_credential, commands::set_active_asr_provider, commands::set_active_llm_provider, + commands::set_active_omni_provider, commands::get_qa_hotkey_label, commands::set_qa_hotkey, commands::set_selection_polish_hotkey, @@ -357,6 +359,7 @@ macro_rules! app_invoke_handler_mobile { $crate::commands::read_credential, $crate::commands::set_active_asr_provider, $crate::commands::set_active_llm_provider, + $crate::commands::set_active_omni_provider, $crate::commands::validate_provider_credentials, $crate::commands::list_provider_models, $crate::commands::list_history, @@ -917,11 +920,7 @@ fn build_microphone_tray_menu>( // CoreAudio device enumeration can block inside AudioUnitSetProperty while AppKit is // finishing launch. Tray menus must be built on the main thread, so only consume the // cache here; the watcher below owns every potentially blocking enumeration. - let devices = app - .state::() - .0 - .lock() - .clone(); + let devices = app.state::().0.lock().clone(); let selected_available = selected.trim().is_empty() || devices.iter().any(|device| device.name == selected); @@ -1066,8 +1065,10 @@ fn start_tray_microphone_watcher(app: AppHandle) { // Linux 无原生路径，返回 false，纯靠下面的慢速兜底。 // 注册失败（OSStatus≠0 / RegisterEndpoint Err）只 warn，不 panic——兜底轮询保证 // 三平台都「永远能检测到设备」。 - let native_registered = - device_watch::spawn_native_watcher(app.clone(), make_microphone_change_handler(app.clone())); + let native_registered = device_watch::spawn_native_watcher( + app.clone(), + make_microphone_change_handler(app.clone()), + ); if native_registered { log::info!("[tray] OS native microphone device watcher registered"); } else { @@ -1185,12 +1186,7 @@ fn apply_windows_caption_theme(window: &tauri::WebviewWindow, dar &immersive_dark, "immersive dark mode", ); - set_dwm_window_attribute( - hwnd, - DWMWA_CAPTION_COLOR, - &caption_color, - "caption color", - ); + set_dwm_window_attribute(hwnd, DWMWA_CAPTION_COLOR, &caption_color, "caption color"); set_dwm_window_attribute(hwnd, DWMWA_TEXT_COLOR, &text_color, "text color"); set_dwm_window_attribute(hwnd, DWMWA_BORDER_COLOR, &border_color, "border color"); } @@ -1676,10 +1672,7 @@ fn bottom_visual_position( #[cfg_attr(not(target_os = "macos"), allow(dead_code))] fn frame_contains_point(frame: LogicalMonitorFrame, x: f64, y: f64) -> bool { - x >= frame.x - && x < frame.x + frame.width - && y >= frame.y - && y < frame.y + frame.height + x >= frame.x && x < frame.x + frame.width && y >= frame.y && y < frame.y + frame.height } #[cfg_attr(not(target_os = "macos"), allow(dead_code))] @@ -1888,11 +1881,8 @@ mod macos_capsule_ax { unsafe fn cfstring_from_static(bytes_with_nul: &[u8]) -> Option { let cstr = CStr::from_bytes_with_nul(bytes_with_nul).ok()?; - let s = CFStringCreateWithCString( - std::ptr::null(), - cstr.as_ptr(), - K_CF_STRING_ENCODING_UTF8, - ); + let s = + CFStringCreateWithCString(std::ptr::null(), cstr.as_ptr(), K_CF_STRING_ENCODING_UTF8); if s.is_null() { None } else { @@ -2209,7 +2199,10 @@ fn make_chat_window_panel_macos(window: &tauri::WebviewWindow /// 解法是把 NSWindow 的 `movableByWindowBackground` 打开——这条路径不依赖窗口是否成为 /// key window，跟 Spotlight / Raycast 的浮窗是同一手法。设一次就够，整个生命周期保持。 #[cfg(target_os = "macos")] -fn make_chat_window_draggable_macos(window: &tauri::WebviewWindow, tag: &str) { +fn make_chat_window_draggable_macos( + window: &tauri::WebviewWindow, + tag: &str, +) { use objc2::msg_send; use objc2::runtime::{AnyObject, Bool}; let Ok(handle) = window.ns_window() else { @@ -2250,8 +2243,9 @@ fn ensure_qa_window(app: &AppHandle) -> Option(app: &AppHandle) -> Option(app: &AppHandle) -> Option> { +fn ensure_less_computer_window( + app: &AppHandle, +) -> Option> { if let Some(w) = app.get_webview_window("less-computer") { return Some(w); } @@ -2446,7 +2442,11 @@ pub(crate) fn show_selection_polish_preview(app: &AppHandle f64 { mod tests { use super::{ bottom_center_position, bottom_visual_position, capsule_height_for_qa, - capsule_visual_height, capsule_window_bounds, clamp_to_monitor, logical_monitor_frame, - frame_contains_point, frame_distance_to_point_squared, parse_tray_polish_mode_id, + capsule_visual_height, capsule_window_bounds, clamp_to_monitor, frame_contains_point, + frame_distance_to_point_squared, logical_monitor_frame, parse_tray_polish_mode_id, rotate_log_if_too_large, tray_polish_mode_menu_entries, tray_style_menu_enabled, LogicalMonitorFrame, LOG_ROTATE_LIMIT_BYTES, }; @@ -3007,10 +3007,7 @@ mod tests { assert_eq!(frame_distance_to_point_squared(frame, 100.0, -100.0), 0.0); assert_eq!(frame_distance_to_point_squared(frame, 100.0, 20.0), 400.0); - assert_eq!( - frame_distance_to_point_squared(frame, -10.0, -910.0), - 200.0 - ); + assert_eq!(frame_distance_to_point_squared(frame, -10.0, -910.0), 200.0); } #[test] diff --git a/openless-all/app/src-tauri/src/llm_gemini.rs b/openless-all/app/src-tauri/src/llm_gemini.rs index a1e524d95..9f9f3e4a5 100644 --- a/openless-all/app/src-tauri/src/llm_gemini.rs +++ b/openless-all/app/src-tauri/src/llm_gemini.rs @@ -15,6 +15,7 @@ use std::time::Duration; +use base64::Engine; use serde_json::{json, Value}; use crate::polish::{ @@ -158,6 +159,33 @@ impl GeminiProvider { Ok(clean_polish_output(&raw)) } + /// 多模态（Omni）识别管线（issue #902）的 Gemini 通道：音频 + 提示词一次调用。 + /// `wav_bytes` 为 `Some` 时以 `inlineData(audio/wav)` 追加到 user parts（已是 + /// 编码好的 WAV 文件字节，PCM→WAV 的转换由 omni 层统一完成）； + /// `None` 时退化为纯文本调用（选区润色 / 历史重润色等文本管线复用同一通道， + /// 读取的是 omni 命名空间的凭据，与传统 LLM 配置隔离）。 + pub(crate) async fn complete_omni( + &self, + system_prompt: &str, + user_text: &str, + wav_bytes: Option<&[u8]>, + ) -> Result { + let contents = omni_gemini_contents(user_text, wav_bytes); + let body = self.build_generate_body(system_prompt, contents); + let url = generate_content_url(&self.config.base_url, &self.config.model); + + log::info!( + "[omni] POST {} provider=gemini model={} audio={}", + crate::net::sanitized_url_for_logs(&url), + self.config.model, + wav_bytes.is_some() + ); + + let body_text = self.send_unary(&url, &body).await?; + let raw = extract_assistant_content(&body_text)?; + Ok(clean_polish_output(&raw)) + } + /// 划词语音问答的流式回答。Gemini 原生 SSE: `:streamGenerateContent?alt=sse`， /// 每个 `data: {...}` 帧里 `candidates[0].content.parts[0].text` 是 delta； /// 流结束没有 `[DONE]` sentinel，stream 自然终止。 @@ -429,6 +457,22 @@ fn build_polish_history_contents( contents } +/// Gemini 多模态调用的一轮 user contents：文本 part 恒在首位，音频 part 可选。 +/// `wav_bytes` 是编码好的 WAV 文件字节，base64 后经 `inlineData(audio/wav)` 下发。 +fn omni_gemini_contents(user_text: &str, wav_bytes: Option<&[u8]>) -> Vec { + let mut parts = vec![json!({ "text": user_text })]; + if let Some(wav) = wav_bytes { + let data = base64::engine::general_purpose::STANDARD.encode(wav); + parts.push(json!({ + "inlineData": { + "mimeType": "audio/wav", + "data": data, + } + })); + } + vec![json!({ "role": "user", "parts": parts })] +} + /// QA chat messages → Gemini contents：assistant role 重命名为 model。 /// QaChatMessage.role 在 polish.rs OpenAI 路径里是 `"user" | "assistant"`； /// 这里把 `assistant` 翻成 Gemini 的 `model`，其它原样保留。 diff --git a/openless-all/app/src-tauri/src/omni.rs b/openless-all/app/src-tauri/src/omni.rs new file mode 100644 index 000000000..1a3da6e4a --- /dev/null +++ b/openless-all/app/src-tauri/src/omni.rs @@ -0,0 +1,481 @@ +//! 多模态（Omni）识别管线（issue #902）的模型通道。 +//! +//! 与 `polish.rs` 的 LLM 客户端不同：这里接收「系统提示词 + 用户文本 + 可选音频」， +//! 让模型一步基于音频与词典/提示词直接输出最终文本，替代「ASR 转写 + LLM 润色」 +//! 两段式管线。凭据读取独立 `omni` 命名空间，与 asr/llm 配置完全隔离。 +//! +//! 通道： +//! - OpenAI 兼容 chat completions：user content 的 `input_audio` part 携带 base64 WAV； +//! - Gemini 原生 generateContent：`inlineData(audio/wav)` part（复用 `llm_gemini.rs`）。 + +use std::collections::HashMap; + +use base64::Engine; +use serde_json::{json, Value}; + +use crate::polish::{ + append_utf8_sse_chunk, apply_openai_compatible_thinking_control, chat_completions_url, + extract_assistant_content, finish_utf8_sse_chunks, http_client_builder, + openai_model_is_gpt5_family, safe_str_slice, send_with_transient_retry, LLMError, +}; + +pub const OMNI_GEMINI_PROVIDER_ID: &str = "gemini"; +/// Omni 请求默认超时（秒）。比普通文本润色长：base64 WAV 上传 + 音频模型生成。 +const OMNI_DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 90; +const BODY_PREVIEW_LIMIT: usize = 200; + +#[derive(Clone, Debug)] +pub struct OmniConfig { + pub provider_id: String, + pub base_url: String, + pub api_key: String, + pub model: String, + pub extra_headers: HashMap, + pub temperature: Option, + pub thinking_enabled: bool, +} + +impl OmniConfig { + pub fn is_gemini(&self) -> bool { + self.provider_id.trim() == OMNI_GEMINI_PROVIDER_ID + || self.base_url.contains("generativelanguage.googleapis.com") + } +} + +/// 一次 Omni 调用的构建时快照（provider id + model），落历史归因用。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OmniCallLabel { + pub provider: String, + pub model: String, +} + +/// OpenAI 兼容 chat completions 通道（`input_audio` 音频 part）。 +pub struct OpenAICompatibleOmni { + config: OmniConfig, + client: reqwest::Client, +} + +impl OpenAICompatibleOmni { + pub fn new(config: OmniConfig) -> Self { + // 与 OpenAICompatibleLLMProvider 同款：按 (超时, 是否绕过代理) 缓存连接池， + // 跨句子复用 TLS 握手。代理开关切换时 net 缓存会清空重建。 + let timeout = OMNI_DEFAULT_REQUEST_TIMEOUT_SECS; + let no_proxy = + crate::net::should_bypass_proxy(&config.base_url, crate::net::use_system_proxy()); + let base_url = config.base_url.clone(); + let client = crate::net::cached_client((timeout, no_proxy), || { + http_client_builder(&base_url, timeout) + .build() + .unwrap_or_else(|_| reqwest::Client::new()) + }); + Self { config, client } + } + + fn omni_body(&self, stream: bool, messages: Vec) -> Value { + let mut body = json!({ + "model": self.config.model, + "stream": stream, + "messages": messages, + }); + if let Some(temperature) = self.config.temperature { + // OpenAI 官方 gpt-5 系列只接受默认 temperature=1（issue #857），同润色路径。 + if !(self.config.provider_id.trim() == "openai" + && openai_model_is_gpt5_family(&self.config.model)) + { + body["temperature"] = json!(temperature); + } + } + apply_openai_compatible_thinking_control( + &mut body, + &self.config.provider_id, + &self.config.base_url, + &self.config.model, + self.config.thinking_enabled, + ); + body + } + + fn build_messages( + &self, + system_prompt: &str, + user_text: &str, + wav_bytes: Option<&[u8]>, + ) -> Vec { + let user_content = match wav_bytes { + Some(wav) => { + let data = base64::engine::general_purpose::STANDARD.encode(wav); + let mut parts = vec![json!({ + "type": "input_audio", + "input_audio": { "data": data, "format": "wav" }, + })]; + if !user_text.trim().is_empty() { + parts.push(json!({ "type": "text", "text": user_text })); + } + Value::Array(parts) + } + None => json!(user_text), + }; + vec![ + json!({ "role": "system", "content": system_prompt }), + json!({ "role": "user", "content": user_content }), + ] + } + + async fn send_unary(&self, url: &str, body: &Value) -> Result { + let mut request = self + .client + .post(url) + .header("Content-Type", "application/json"); + if !self.config.api_key.trim().is_empty() { + request = request.header("Authorization", format!("Bearer {}", self.config.api_key)); + } + for (key, value) in &self.config.extra_headers { + request = request.header(key.as_str(), value.as_str()); + } + let request = request.json(body); + let response = send_with_transient_retry(request).await?; + let status = response.status(); + let body_text = response + .text() + .await + .map_err(crate::polish::llm_error_from_reqwest)?; + let preview_end = BODY_PREVIEW_LIMIT.min(body_text.len()); + let preview = safe_str_slice(&body_text, preview_end); + log::info!("[omni] HTTP {} body={}", status.as_u16(), preview); + if !status.is_success() { + return Err(LLMError::InvalidResponse { + status: status.as_u16(), + body: preview.to_string(), + }); + } + extract_assistant_content(&body_text) + } + + async fn send_streaming( + &self, + url: &str, + body: &Value, + on_delta: F, + should_cancel: C, + ) -> Result + where + F: Fn(&str) + Send + Sync, + C: Fn() -> bool + Send + Sync, + { + let mut request = self + .client + .post(url) + .header("Content-Type", "application/json") + .header("Accept", "text/event-stream"); + if !self.config.api_key.trim().is_empty() { + request = request.header("Authorization", format!("Bearer {}", self.config.api_key)); + } + for (key, value) in &self.config.extra_headers { + request = request.header(key.as_str(), value.as_str()); + } + let request = request.json(body); + let response = send_with_transient_retry(request).await?; + let status = response.status(); + if !status.is_success() { + let body_text = response + .text() + .await + .map_err(crate::polish::llm_error_from_reqwest)?; + let preview_end = BODY_PREVIEW_LIMIT.min(body_text.len()); + let preview = safe_str_slice(&body_text, preview_end); + log::error!("[omni] streaming HTTP {} body={}", status.as_u16(), preview); + return Err(LLMError::InvalidResponse { + status: status.as_u16(), + body: preview.to_string(), + }); + } + + // SSE 流解析与 polish 路径同款：一帧 = 若干行，`\n\n` 分隔， + // 每行 `data: {...}` / `data: [DONE]`。 + let mut response = response; + let mut buffer = String::new(); + let mut utf8_pending: Vec = Vec::new(); + let mut full_text = String::new(); + let mut cancelled = false; + loop { + if should_cancel() { + log::info!("[omni] stream cancelled by caller; breaking SSE loop"); + cancelled = true; + break; + } + let chunk_opt = response + .chunk() + .await + .map_err(crate::polish::llm_error_from_reqwest)?; + let Some(chunk) = chunk_opt else { break }; + append_utf8_sse_chunk(&mut buffer, &mut utf8_pending, &chunk)?; + while let Some(idx) = buffer.find("\n\n") { + let event = buffer[..idx].to_string(); + buffer.drain(..idx + 2); + for line in event.lines() { + let Some(payload) = line + .strip_prefix("data: ") + .or_else(|| line.strip_prefix("data:")) + else { + continue; + }; + let payload = payload.trim(); + if payload.is_empty() || payload == "[DONE]" { + continue; + } + let value: Value = match serde_json::from_str(payload) { + Ok(value) => value, + Err(error) => { + log::warn!( + "[omni] SSE parse skip: {error}; payload preview: {}", + safe_str_slice(payload, 80) + ); + continue; + } + }; + if let Some(delta) = value["choices"][0]["delta"]["content"].as_str() { + if !delta.is_empty() { + full_text.push_str(delta); + on_delta(delta); + } + } + } + } + } + if !cancelled { + finish_utf8_sse_chunks(&mut buffer, &mut utf8_pending)?; + } + log::info!( + "[omni] stream done; total chars={}", + full_text.chars().count() + ); + if full_text.is_empty() { + return Err(LLMError::InvalidResponse { + status: 200, + body: "empty omni stream".to_string(), + }); + } + Ok(full_text) + } + + pub(crate) async fn complete( + &self, + system_prompt: &str, + user_text: &str, + wav_bytes: Option<&[u8]>, + ) -> Result { + let messages = self.build_messages(system_prompt, user_text, wav_bytes); + let body = self.omni_body(false, messages); + let url = chat_completions_url(&self.config.base_url); + log::info!( + "[omni] POST {} provider={} model={} audio={}", + crate::net::sanitized_url_for_logs(&url), + self.config.provider_id, + self.config.model, + wav_bytes.is_some() + ); + self.send_unary(&url, &body).await + } + + pub(crate) async fn complete_streaming( + &self, + system_prompt: &str, + user_text: &str, + wav_bytes: Option<&[u8]>, + on_delta: F, + should_cancel: C, + ) -> Result + where + F: Fn(&str) + Send + Sync, + C: Fn() -> bool + Send + Sync, + { + let messages = self.build_messages(system_prompt, user_text, wav_bytes); + let body = self.omni_body(true, messages); + let url = chat_completions_url(&self.config.base_url); + log::info!( + "[omni] POST {} provider={} model={} audio={} stream=true", + crate::net::sanitized_url_for_logs(&url), + self.config.provider_id, + self.config.model, + wav_bytes.is_some() + ); + self.send_streaming(&url, &body, on_delta, should_cancel) + .await + } +} + +/// 多模态通道统一入口：按配置路由到 Gemini 原生或 OpenAI 兼容客户端。 +pub enum OmniProvider { + Gemini { + provider: crate::llm_gemini::GeminiProvider, + label: OmniCallLabel, + }, + OpenAI(OpenAICompatibleOmni), +} + +impl OmniProvider { + pub fn new(config: OmniConfig) -> Self { + if config.is_gemini() { + let label = OmniCallLabel { + provider: config.provider_id.clone(), + model: config.model.clone(), + }; + let gemini_config = crate::llm_gemini::GeminiConfig::new( + config.api_key.clone(), + config.model.clone(), + config.base_url.clone(), + ) + .with_thinking_enabled(config.thinking_enabled); + let mut gemini_config = gemini_config; + if let Some(temperature) = config.temperature { + gemini_config.temperature = temperature; + } + Self::Gemini { + provider: crate::llm_gemini::GeminiProvider::new(gemini_config), + label, + } + } else { + Self::OpenAI(OpenAICompatibleOmni::new(config)) + } + } + + pub fn call_label(&self) -> OmniCallLabel { + match self { + Self::Gemini { label, .. } => label.clone(), + Self::OpenAI(provider) => OmniCallLabel { + provider: provider.config.provider_id.clone(), + model: provider.config.model.clone(), + }, + } + } + + /// 一次性调用：音频 + 提示词一步输出最终文本；无音频时为纯文本（文本管线复用）。 + pub async fn complete( + &self, + system_prompt: &str, + user_text: &str, + wav_bytes: Option<&[u8]>, + ) -> Result { + match self { + Self::Gemini { provider, .. } => { + provider + .complete_omni(system_prompt, user_text, wav_bytes) + .await + } + Self::OpenAI(provider) => provider.complete(system_prompt, user_text, wav_bytes).await, + } + } + + /// 流式输出。OpenAI 兼容通道按 SSE 逐字回调；Gemini 通道 v1 一次性返回后 + /// 以单次 `on_delta` 回调完整文本（与批准方案的「Gemini 回退一次性」一致）。 + pub async fn complete_streaming( + &self, + system_prompt: &str, + user_text: &str, + wav_bytes: Option<&[u8]>, + on_delta: F, + should_cancel: C, + ) -> Result + where + F: Fn(&str) + Send + Sync, + C: Fn() -> bool + Send + Sync, + { + match self { + Self::Gemini { provider, .. } => { + let text = provider + .complete_omni(system_prompt, user_text, wav_bytes) + .await?; + on_delta(&text); + Ok(text) + } + Self::OpenAI(provider) => { + provider + .complete_streaming( + system_prompt, + user_text, + wav_bytes, + on_delta, + should_cancel, + ) + .await + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> OmniConfig { + OmniConfig { + provider_id: "openai".into(), + base_url: "https://api.openai.com/v1".into(), + api_key: "sk-test".into(), + model: "gpt-4o-audio-preview".into(), + extra_headers: HashMap::new(), + temperature: Some(0.3), + thinking_enabled: false, + } + } + + #[test] + fn build_messages_embeds_wav_as_input_audio_part() { + let provider = OpenAICompatibleOmni::new(config()); + let messages = provider.build_messages("system-prompt", "", Some(&[1u8, 2, 3, 4])); + assert_eq!(messages.len(), 2); + assert_eq!(messages[0]["role"], "system"); + assert_eq!(messages[0]["content"], "system-prompt"); + assert_eq!(messages[1]["role"], "user"); + let parts = messages[1]["content"].as_array().expect("audio parts"); + assert_eq!(parts[0]["type"], "input_audio"); + assert_eq!(parts[0]["input_audio"]["format"], "wav"); + let data = parts[0]["input_audio"]["data"] + .as_str() + .expect("base64 data"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(data) + .expect("valid base64"); + assert_eq!(decoded, vec![1u8, 2, 3, 4]); + // 空 user_text 时不追加多余 text part。 + assert_eq!(parts.len(), 1); + } + + #[test] + fn build_messages_text_only_when_no_audio() { + let provider = OpenAICompatibleOmni::new(config()); + let messages = provider.build_messages("system", "你好", None); + assert_eq!(messages[1]["content"], "你好"); + } + + #[test] + fn build_messages_appends_text_part_alongside_audio() { + let provider = OpenAICompatibleOmni::new(config()); + let messages = provider.build_messages("system", "翻译成中文", Some(&[0u8; 8])); + let parts = messages[1]["content"].as_array().expect("audio parts"); + assert_eq!(parts.len(), 2); + assert_eq!(parts[1]["type"], "text"); + assert_eq!(parts[1]["text"], "翻译成中文"); + } + + #[test] + fn omni_body_has_stream_model_and_temperature() { + let provider = OpenAICompatibleOmni::new(config()); + let body = provider.omni_body(true, vec![json!({"role": "user", "content": "x"})]); + assert_eq!(body["stream"], true); + assert_eq!(body["model"], "gpt-4o-audio-preview"); + // temperature 以 f32 存（0.3f32 序列化后是 0.30000001192092896），用容差比较。 + assert!((body["temperature"].as_f64().unwrap() - 0.3).abs() < 1e-6); + } + + #[test] + fn omni_gemini_routing_uses_provider_id_or_base_url() { + assert!(config().is_gemini() == false); + let mut gemini = config(); + gemini.provider_id = "gemini".into(); + assert!(gemini.is_gemini()); + let mut via_url = config(); + via_url.base_url = "https://generativelanguage.googleapis.com/v1beta".into(); + assert!(via_url.is_gemini()); + } +} diff --git a/openless-all/app/src-tauri/src/persistence/credentials.rs b/openless-all/app/src-tauri/src/persistence/credentials.rs index e1d80a235..2efe366a8 100644 --- a/openless-all/app/src-tauri/src/persistence/credentials.rs +++ b/openless-all/app/src-tauri/src/persistence/credentials.rs @@ -127,6 +127,10 @@ struct CredsRoot { active: CredsActive, #[serde(default)] providers: CredsProviders, + /// 多模态识别管线（issue #902）专用凭据命名空间，与 asr/llm 完全隔离： + /// 运行时只在 `pipeline_mode == multimodal` 时读取，切换模式不删除。 + #[serde(default)] + omni: CredsOmni, #[serde(default, skip_serializing_if = "CredsMarketplace::is_empty")] marketplace: CredsMarketplace, } @@ -174,6 +178,50 @@ struct CredsProviders { llm: HashMap, } +/// 多模态（Omni）模型配置：一个 active provider + 按 provider 隔离的 entry。 +/// entry 字段形状与 LLM 对齐（API Key / Base URL / Model / 温度 / 额外请求头）， +/// 但存放在独立命名空间，绝不与 `providers.llm` 共享槽位。 +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +struct CredsOmni { + #[serde(default = "creds_default_omni")] + active: String, + #[serde(default)] + providers: HashMap, +} + +fn creds_default_omni() -> String { + "custom".into() +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[allow(non_snake_case)] +struct CredsOmniEntry { + #[serde(skip_serializing_if = "Option::is_none")] + apiKey: Option, + #[serde(skip_serializing_if = "Option::is_none")] + baseURL: Option, + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + extraHeaders: Option>, +} + +impl CredsOmniEntry { + fn is_empty(&self) -> bool { + self.apiKey.as_deref().unwrap_or("").is_empty() + && self.baseURL.as_deref().unwrap_or("").is_empty() + && self.model.as_deref().unwrap_or("").is_empty() + && self.temperature.is_none() + && self + .extraHeaders + .as_ref() + .map(|h| h.is_empty()) + .unwrap_or(true) + } +} + #[derive(Debug, Serialize, Deserialize, Default, Clone)] #[allow(non_snake_case)] struct CredsMarketplace { @@ -290,6 +338,14 @@ fn active_llm_extra_headers(root: &CredsRoot) -> HashMap { .unwrap_or_default() } +fn active_omni_extra_headers(root: &CredsRoot) -> HashMap { + root.omni + .providers + .get(&root.omni.active) + .and_then(|entry| entry.extraHeaders.clone()) + .unwrap_or_default() +} + fn is_valid_llm_temperature(temperature: f64) -> bool { temperature.is_finite() && (0.0..=2.0).contains(&temperature) } @@ -321,6 +377,33 @@ fn active_llm_extra_headers_json(root: &CredsRoot) -> Result> { .context("encode LLM extra headers") } +fn active_omni_extra_headers_json(root: &CredsRoot) -> Result> { + let headers = active_omni_extra_headers(root); + if headers.is_empty() { + return Ok(None); + } + let ordered = headers.into_iter().collect::>(); + serde_json::to_string(&ordered) + .map(Some) + .context("encode omni extra headers") +} + +fn active_omni_temperature_value(root: &CredsRoot) -> Option { + root.omni + .providers + .get(&root.omni.active) + .and_then(|entry| entry.temperature) + .filter(|temperature| is_valid_llm_temperature(*temperature)) +} + +fn active_omni_temperature(root: &CredsRoot) -> Option { + active_omni_temperature_value(root).map(|temperature| temperature as f32) +} + +fn active_omni_temperature_string(root: &CredsRoot) -> Option { + active_omni_temperature_value(root).map(|temperature| temperature.to_string()) +} + fn parse_extra_headers_json(value: &str) -> Result> { let trimmed = value.trim(); if trimmed.is_empty() { @@ -540,13 +623,13 @@ fn load_android_credentials_from_source_with_crypto( ReadOutcome::Legacy(bytes) => (bytes, true), ReadOutcome::Plaintext(bytes) => (bytes, false), }; - let root = serde_json::from_slice::(&bytes) - .context("parse Android credential payload")?; + let root = + serde_json::from_slice::(&bytes).context("parse Android credential payload")?; let cleaned = android_persistable_credentials(&root); let contained_marketplace_token = lookup_marketplace_github_token(&root).is_some(); if needs_rewrite && contained_marketplace_token { - let sanitized = serde_json::to_vec(&cleaned) - .context("encode bearer-free Android legacy payload")?; + let sanitized = + serde_json::to_vec(&cleaned).context("encode bearer-free Android legacy payload")?; super::android_credentials::rewrite_legacy_without_bearer(source_path, &sanitized) .map_err(anyhow::Error::new) .context("scrub Marketplace bearer before Android Keystore migration")?; @@ -636,6 +719,7 @@ fn clean_credentials(root: &CredsRoot) -> CredsRoot { let mut cleaned = root.clone(); cleaned.providers.asr.retain(|_, v| !v.is_empty()); cleaned.providers.llm.retain(|_, v| !v.is_empty()); + cleaned.omni.providers.retain(|_, v| !v.is_empty()); cleaned } @@ -1124,6 +1208,7 @@ fn save_credentials(root: &CredsRoot) -> Result<()> { fn lookup_account(root: &CredsRoot, account: CredentialAccount) -> Option { let asr = root.providers.asr.get(&root.active.asr); let llm = root.providers.llm.get(&root.active.llm); + let omni = root.omni.providers.get(&root.omni.active); let pick = |s: &Option| s.as_ref().filter(|v| !v.is_empty()).cloned(); match account { CredentialAccount::VolcengineAppKey => { @@ -1143,12 +1228,16 @@ fn lookup_account(root: &CredsRoot, account: CredentialAccount) -> Option asr.and_then(|e| pick(&e.advancedConfig)), CredentialAccount::XfyunAppId => asr.and_then(|e| pick(&e.xfyunAppId)), CredentialAccount::XfyunApiKey => asr.and_then(|e| pick(&e.xfyunApiKey)), + CredentialAccount::OmniApiKey => omni.and_then(|e| pick(&e.apiKey)), + CredentialAccount::OmniEndpoint => omni.and_then(|e| pick(&e.baseURL)), + CredentialAccount::OmniModel => omni.and_then(|e| pick(&e.model)), } } fn write_account(root: &mut CredsRoot, account: CredentialAccount, value: Option) { let asr_id = root.active.asr.clone(); let llm_id = root.active.llm.clone(); + let omni_id = root.omni.active.clone(); let normalized = value.and_then(|v| if v.is_empty() { None } else { Some(v) }); match account { CredentialAccount::VolcengineAppKey => { @@ -1211,6 +1300,18 @@ fn write_account(root: &mut CredsRoot, account: CredentialAccount, value: Option let entry = root.providers.asr.entry(asr_id).or_default(); entry.xfyunApiKey = normalized; } + CredentialAccount::OmniApiKey => { + let entry = root.omni.providers.entry(omni_id).or_default(); + entry.apiKey = normalized; + } + CredentialAccount::OmniEndpoint => { + let entry = root.omni.providers.entry(omni_id).or_default(); + entry.baseURL = normalized; + } + CredentialAccount::OmniModel => { + let entry = root.omni.providers.entry(omni_id).or_default(); + entry.model = normalized; + } } } @@ -1239,6 +1340,12 @@ pub enum CredentialAccount { XfyunAppId, /// 讯飞实时语音转写 APIKey。 XfyunApiKey, + /// 多模态（Omni）模型的 API Key。仅多模态管线读取。 + OmniApiKey, + /// 多模态（Omni）模型的 Base URL。 + OmniEndpoint, + /// 多模态（Omni）模型的 model id。 + OmniModel, } impl CredentialAccount { @@ -1262,6 +1369,9 @@ impl CredentialAccount { CredentialAccount::AsrAdvancedConfig => "asr.advanced_config", CredentialAccount::XfyunAppId => "xfyun.app_id", CredentialAccount::XfyunApiKey => "xfyun.api_key", + CredentialAccount::OmniApiKey => "omni.api_key", + CredentialAccount::OmniEndpoint => "omni.endpoint", + CredentialAccount::OmniModel => "omni.model", } } @@ -1282,6 +1392,9 @@ impl CredentialAccount { CredentialAccount::AsrAdvancedConfig, CredentialAccount::XfyunAppId, CredentialAccount::XfyunApiKey, + CredentialAccount::OmniApiKey, + CredentialAccount::OmniEndpoint, + CredentialAccount::OmniModel, ] } } @@ -1302,6 +1415,10 @@ pub struct CredentialsSnapshot { pub ark_api_key: Option, pub ark_model_id: Option, pub ark_endpoint: Option, + pub active_omni_provider: String, + pub omni_api_key: Option, + pub omni_endpoint: Option, + pub omni_model: Option, } /// 凭据存储——系统凭据库；旧 JSON 文件只作为迁移来源。 @@ -1465,6 +1582,68 @@ impl CredentialsVault { load_credentials().active.llm } + pub fn get_active_omni() -> String { + let _guard = credentials_lock().lock(); + load_credentials().omni.active + } + + pub fn set_active_omni_provider(id: &str) -> Result<()> { + let _guard = credentials_lock().lock(); + let mut root = load_credentials_for_update()?; + root.omni.active = id.to_string(); + save_credentials(&root) + } + + pub fn get_active_omni_extra_headers() -> HashMap { + let _guard = credentials_lock().lock(); + active_omni_extra_headers(&load_credentials()) + } + + pub fn get_active_omni_extra_headers_json() -> Result> { + let _guard = credentials_lock().lock(); + active_omni_extra_headers_json(&load_credentials()) + } + + pub fn get_active_omni_temperature() -> Option { + let _guard = credentials_lock().lock(); + active_omni_temperature(&load_credentials()) + } + + pub fn get_active_omni_temperature_string() -> Option { + let _guard = credentials_lock().lock(); + active_omni_temperature_string(&load_credentials()) + } + + pub fn set_active_omni_temperature(value: &str) -> Result<()> { + let _guard = credentials_lock().lock(); + let temperature = parse_llm_temperature(value)?; + let mut root = load_credentials_for_update()?; + let entry = root + .omni + .providers + .entry(root.omni.active.clone()) + .or_default(); + entry.temperature = temperature; + save_credentials(&root) + } + + pub fn set_active_omni_extra_headers_json(value: &str) -> Result<()> { + let _guard = credentials_lock().lock(); + let headers = parse_extra_headers_json(value)?; + let mut root = load_credentials_for_update()?; + let entry = root + .omni + .providers + .entry(root.omni.active.clone()) + .or_default(); + entry.extraHeaders = if headers.is_empty() { + None + } else { + Some(headers) + }; + save_credentials(&root) + } + pub fn get_active_llm_extra_headers() -> HashMap { let _guard = credentials_lock().lock(); active_llm_extra_headers(&load_credentials()) @@ -1489,7 +1668,11 @@ impl CredentialsVault { let _guard = credentials_lock().lock(); let temperature = parse_llm_temperature(value)?; let mut root = load_credentials_for_update()?; - let entry = root.providers.llm.entry(root.active.llm.clone()).or_default(); + let entry = root + .providers + .llm + .entry(root.active.llm.clone()) + .or_default(); entry.temperature = temperature; save_credentials(&root) } @@ -1498,7 +1681,11 @@ impl CredentialsVault { let _guard = credentials_lock().lock(); let headers = parse_extra_headers_json(value)?; let mut root = load_credentials_for_update()?; - let entry = root.providers.llm.entry(root.active.llm.clone()).or_default(); + let entry = root + .providers + .llm + .entry(root.active.llm.clone()) + .or_default(); entry.extraHeaders = if headers.is_empty() { None } else { @@ -1524,12 +1711,18 @@ impl CredentialsVault { ark_api_key: lookup_account(&root, CredentialAccount::ArkApiKey), ark_model_id: lookup_account(&root, CredentialAccount::ArkModelId), ark_endpoint: lookup_account(&root, CredentialAccount::ArkEndpoint), + active_omni_provider: root.omni.active.clone(), + omni_api_key: lookup_account(&root, CredentialAccount::OmniApiKey), + omni_endpoint: lookup_account(&root, CredentialAccount::OmniEndpoint), + omni_model: lookup_account(&root, CredentialAccount::OmniModel), } } } #[cfg(test)] mod tests { + #[cfg(not(windows))] + use super::load_android_credentials_from_source_with_crypto; use super::{ android_persistable_credentials, chunk_json_payload, credentials_cache, get_android_marketplace_token_at, load_android_credentials_from_path, @@ -1539,8 +1732,6 @@ mod tests { write_marketplace_github_token, CredentialAccount, CredsAsrEntry, CredsRoot, MarketplaceGithubToken, KEYRING_CHUNK_MAX_UTF16_UNITS, }; - #[cfg(not(windows))] - use super::load_android_credentials_from_source_with_crypto; use anyhow::anyhow; use parking_lot::Mutex; @@ -1560,6 +1751,50 @@ mod tests { .all(|chunk| chunk.encode_utf16().count() <= KEYRING_CHUNK_MAX_UTF16_UNITS)); } + #[test] + fn omni_accounts_route_to_omni_namespace_only() { + // 多模态（Omni）凭据必须与 LLM/ASR 命名空间完全隔离（issue #902）： + // 写 omni 槽位不影响 ark 槽位；切换 omni active provider 后读到的是 + // 该 provider 自己的 entry，而不是别的 provider 的残留值。 + let mut root = CredsRoot::default(); + root.active.llm = "ark".into(); + root.active.asr = "volcengine".into(); + root.omni.active = "openai".into(); + + write_account( + &mut root, + CredentialAccount::OmniApiKey, + Some("omni-key".into()), + ); + write_account( + &mut root, + CredentialAccount::OmniEndpoint, + Some("https://api.openai.com/v1".into()), + ); + write_account( + &mut root, + CredentialAccount::OmniModel, + Some("gpt-4o-audio-preview".into()), + ); + + assert_eq!( + lookup_account(&root, CredentialAccount::OmniApiKey).as_deref(), + Some("omni-key") + ); + // 传统 LLM / ASR 槽位必须保持为空。 + assert_eq!(lookup_account(&root, CredentialAccount::ArkApiKey), None); + assert_eq!(lookup_account(&root, CredentialAccount::AsrApiKey), None); + + // 切到另一个 omni provider：读不到 openai 的 entry（per-provider 隔离）。 + root.omni.active = "custom".into(); + assert_eq!(lookup_account(&root, CredentialAccount::OmniApiKey), None); + root.omni.active = "openai".into(); + assert_eq!( + lookup_account(&root, CredentialAccount::OmniModel).as_deref(), + Some("gpt-4o-audio-preview") + ); + } + #[test] fn parse_extra_headers_json_rejects_reserved_header_names() { for name in [ @@ -1609,8 +1844,13 @@ mod tests { // 清空即移除该字段，且只影响对应 provider 的 entry。 write_account(&mut root, CredentialAccount::AsrAdvancedConfig, None); - assert_eq!(lookup_account(&root, CredentialAccount::AsrAdvancedConfig), None); - assert!(root.providers.asr["openai-compatible"].advancedConfig.is_none()); + assert_eq!( + lookup_account(&root, CredentialAccount::AsrAdvancedConfig), + None + ); + assert!(root.providers.asr["openai-compatible"] + .advancedConfig + .is_none()); // 旧条目（无 advancedConfig 字段）反序列化为 None，不破坏既有数据。 let legacy: CredsAsrEntry = serde_json::from_str(r#"{"apiKey":"k"}"#).unwrap(); @@ -1736,9 +1976,11 @@ mod tests { assert!(std::fs::read_to_string(&destination_path) .unwrap() .contains("openless-android-credentials")); - assert!(load_android_credentials_from_path_with_crypto(&destination_path, &mut crypto) - .unwrap() - .is_some()); + assert!( + load_android_credentials_from_path_with_crypto(&destination_path, &mut crypto) + .unwrap() + .is_some() + ); std::fs::remove_dir_all(root_dir).unwrap(); } @@ -1800,9 +2042,8 @@ mod tests { ) .unwrap(); let mut crypto = super::super::android_credentials::TestCrypto::default(); - crypto.fail_next_seal = Some( - super::super::android_credentials::CryptoErrorKind::TemporarilyUnavailable, - ); + crypto.fail_next_seal = + Some(super::super::android_credentials::CryptoErrorKind::TemporarilyUnavailable); assert!(load_android_credentials_from_path_with_crypto(&path, &mut crypto).is_err()); let sanitized = std::fs::read(&path).unwrap(); diff --git a/openless-all/app/src-tauri/src/polish.rs b/openless-all/app/src-tauri/src/polish.rs index c5920fe63..248451b20 100644 --- a/openless-all/app/src-tauri/src/polish.rs +++ b/openless-all/app/src-tauri/src/polish.rs @@ -587,7 +587,13 @@ impl OpenAICompatibleLLMProvider { body["temperature"] = json!(temperature); } } - apply_openai_compatible_thinking_control(&mut body, &self.config); + apply_openai_compatible_thinking_control( + &mut body, + &self.config.provider_id, + &self.config.base_url, + &self.config.model, + self.config.thinking_enabled, + ); body } @@ -1211,7 +1217,7 @@ impl CodexOAuthLLMProvider { } } -fn append_utf8_sse_chunk( +pub(crate) fn append_utf8_sse_chunk( buffer: &mut String, pending: &mut Vec, chunk: &[u8], @@ -1220,7 +1226,10 @@ fn append_utf8_sse_chunk( drain_complete_utf8(buffer, pending) } -fn finish_utf8_sse_chunks(buffer: &mut String, pending: &mut Vec) -> Result<(), LLMError> { +pub(crate) fn finish_utf8_sse_chunks( + buffer: &mut String, + pending: &mut Vec, +) -> Result<(), LLMError> { drain_complete_utf8(buffer, pending)?; if pending.is_empty() { Ok(()) @@ -1297,7 +1306,7 @@ fn build_polish_history_messages( messages } -fn chat_completions_url(base_url: &str) -> String { +pub(crate) fn chat_completions_url(base_url: &str) -> String { let trimmed = base_url.trim(); let Ok(mut url) = reqwest::Url::parse(trimmed) else { let fallback = trimmed.trim_end_matches('/'); @@ -1341,7 +1350,7 @@ fn should_retry_transient(is_connect: bool, is_request: bool, is_timeout: bool) /// 对流式 SSE 路径 retry 是安全的：connect / request 类失败发生在 TCP 握手 / HTTP /// 请求写出阶段，response 还没回 → on_delta 必然未被调用 → 不会有「已流式输出的字 /// 被重复」的问题。 -async fn send_with_transient_retry( +pub(crate) async fn send_with_transient_retry( request: reqwest::RequestBuilder, ) -> Result { const RETRY_DELAY_MS: u64 = 500; @@ -1578,41 +1587,43 @@ fn unix_now_secs() -> u64 { .unwrap_or(0) } -fn apply_openai_compatible_thinking_control(body: &mut Value, config: &OpenAICompatibleConfig) { +pub(crate) fn apply_openai_compatible_thinking_control( + body: &mut Value, + provider_id: &str, + base_url: &str, + model: &str, + thinking_enabled: bool, +) { // 优先按 provider_id 预设分派；custom / 未声明 provider 时回退到 base_url 兜底, // 让用户用"自定义"preset 接入 MiniMax 也能正确下发 thinking 控制参数。 - let control = openai_compatible_thinking_control(&config.provider_id) - .or_else(|| openai_compatible_thinking_control_for_base_url(&config.base_url)); + let control = openai_compatible_thinking_control(provider_id) + .or_else(|| openai_compatible_thinking_control_for_base_url(base_url)); match control { Some(ThinkingControl::ReasoningEffort) => { // OpenAI 官方 Chat Completions 只在推理模型族接受 reasoning_effort； // 普通 chat 模型会直接 400。其它兼容渠道按渠道声明继续下发。 - let effort = if config.provider_id.trim() == "openai" { - openai_chat_reasoning_effort(&config.model, config.thinking_enabled) + let effort = if provider_id.trim() == "openai" { + openai_chat_reasoning_effort(model, thinking_enabled) } else { - Some(if config.thinking_enabled { - "medium" - } else { - "low" - }) + Some(if thinking_enabled { "medium" } else { "low" }) }; if let Some(effort) = effort { body["reasoning_effort"] = json!(effort); } } Some(ThinkingControl::EnableThinking) => { - body["enable_thinking"] = json!(config.thinking_enabled); + body["enable_thinking"] = json!(thinking_enabled); } Some(ThinkingControl::OpenRouterReasoning) => { body["reasoning"] = json!({ - "effort": if config.thinking_enabled { "medium" } else { "none" }, + "effort": if thinking_enabled { "medium" } else { "none" }, // OpenLess 的 QA/润色输出只展示最终答案；推理内容即使生成，也不应进 UI。 "exclude": true, }); } Some(ThinkingControl::DeepSeekThinking) => { body["thinking"] = json!({ - "type": if config.thinking_enabled { "enabled" } else { "disabled" }, + "type": if thinking_enabled { "enabled" } else { "disabled" }, }); } // MiniMax OpenAI 兼容 Chat Completions 接受官方 `thinking` 字段，关闭用 @@ -1623,7 +1634,7 @@ fn apply_openai_compatible_thinking_control(body: &mut Value, config: &OpenAICom // 这与 OpenLess 渠道级"按官方参数声明下发"的策略一致,不维护单模型白名单。 Some(ThinkingControl::MiniMaxThinking) => { body["thinking"] = json!({ - "type": if config.thinking_enabled { "adaptive" } else { "disabled" }, + "type": if thinking_enabled { "adaptive" } else { "disabled" }, }); } None => {} @@ -1631,7 +1642,7 @@ fn apply_openai_compatible_thinking_control(body: &mut Value, config: &OpenAICom } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ThinkingControl { +pub(crate) enum ThinkingControl { ReasoningEffort, EnableThinking, OpenRouterReasoning, @@ -1639,7 +1650,7 @@ enum ThinkingControl { MiniMaxThinking, } -fn openai_compatible_thinking_control(provider_id: &str) -> Option { +pub(crate) fn openai_compatible_thinking_control(provider_id: &str) -> Option { match provider_id.trim() { "deepseek" => Some(ThinkingControl::DeepSeekThinking), // provider_id 预设(见 ProvidersSection.tsx::LLM_PRESETS)。 @@ -1660,7 +1671,9 @@ fn openai_compatible_thinking_control(provider_id: &str) -> Option Option { +pub(crate) fn openai_compatible_thinking_control_for_base_url( + base_url: &str, +) -> Option { // 抽 host(不区分大小写),允许带端口。`base_url` 末尾可能带 `/v1`、`/v1/`、 // 甚至 `/v1/chat/completions`——统一取第一个 `/` 段当 host。 let host = base_url @@ -1693,7 +1706,7 @@ fn openai_compatible_thinking_control_for_base_url(base_url: &str) -> Option bool { +pub(crate) fn openai_model_is_gpt5_family(model: &str) -> bool { model .trim() .strip_prefix("openai/") @@ -1724,7 +1737,7 @@ fn openai_chat_reasoning_effort(model: &str, thinking_enabled: bool) -> Option<& } } -fn extract_assistant_content(body: &str) -> Result { +pub(crate) fn extract_assistant_content(body: &str) -> Result { let json: Value = serde_json::from_str(body) .map_err(|e| LLMError::ParseError(format!("not valid JSON: {}", e)))?; let choices = json @@ -2603,7 +2616,13 @@ mod tests { #[test] fn chat_body_omits_temperature_for_openai_gpt5_family() { - for model in ["gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5.5", "openai/gpt-5"] { + for model in [ + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5.5", + "openai/gpt-5", + ] { let provider = OpenAICompatibleLLMProvider::new(OpenAICompatibleConfig::new( "openai", "OpenAI", @@ -3242,7 +3261,10 @@ mod tests { structured.contains("高置信度") && structured.contains("低置信度"), "Structured prompt 缺少置信度分级" ); - assert!(structured.contains("根目录"), "Structured prompt 缺少根目录纠错示例"); + assert!( + structured.contains("根目录"), + "Structured prompt 缺少根目录纠错示例" + ); } #[test] diff --git a/openless-all/app/src-tauri/src/types.rs b/openless-all/app/src-tauri/src/types.rs index b8536cf9d..9bfdcd93c 100644 --- a/openless-all/app/src-tauri/src/types.rs +++ b/openless-all/app/src-tauri/src/types.rs @@ -30,6 +30,29 @@ pub enum PolishMode { Formal, } +/// 识别管线模式（issue #902）：`traditional` = 两段式 ASR + LLM 润色； +/// `multimodal` = 单个多模态模型一步完成「音频 + 提示词 → 最终文本」。 +/// 两套配置在凭据库中完全隔离，运行时只读当前模式，切换不删除另一套配置。 +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum PipelineMode { + #[default] + Traditional, + Multimodal, +} + +fn default_pipeline_mode() -> PipelineMode { + PipelineMode::Traditional +} + +fn default_multimodal_pipeline_enabled() -> bool { + false +} + +fn default_active_omni_provider() -> String { + "custom".into() +} + /// 历史记录的产生来源。旧版 `history.json` 未写入该字段时，按既有听写记录处理。 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] @@ -201,6 +224,11 @@ pub struct DictationSession { /// 本次润色用的 LLM 模型 id。Raw 直通时 None。 #[serde(default)] pub llm_model: Option, + /// 本次会话走的识别管线模式（"multimodal" / 缺失 = 传统两段式）。 + /// 多模态会话 `asr_provider/asr_model` 为空，`llm_provider/llm_model` + /// 记实际调用的多模态模型，`polish_ms` 记该调用的耗时。 + #[serde(default)] + pub pipeline_mode: Option, /// 松键后「等待转写结果」的实测耗时（毫秒）。流式 ASR 大部分识别在录音期间已完成， /// 这里量的是用户感知的收尾延迟；批式 ASR 则是完整转写耗时。 #[serde(default)] @@ -702,6 +730,17 @@ pub struct UserPreferences { pub microphone_device_name: String, pub active_asr_provider: String, // "volcengine" | "apple-speech" | ... pub active_llm_provider: String, // "ark" | "openai" | ... + /// 识别管线模式（实验性，issue #902）。`multimodal` 时各语音管线改用 + /// 单独隔离的多模态模型配置（`omni.*` 凭据命名空间），不再读 ASR/LLM 两套。 + #[serde(default = "default_pipeline_mode")] + pub pipeline_mode: PipelineMode, + /// 「多模态识别管线」实验性功能总开关（高级设置）。关闭时一切行为与旧版一致。 + #[serde(default = "default_multimodal_pipeline_enabled")] + pub multimodal_pipeline_enabled: bool, + /// 多模态（Omni）模型当前激活的 provider id（镜像凭据库 `omni.active`， + /// 供设置页初始化下拉；运行时权威仍在 CredentialsVault）。 + #[serde(default = "default_active_omni_provider")] + pub active_omni_provider: String, /// LLM 思考模式开关。默认 false 以保持既有「尽量关闭思考」行为； /// Gemini 走原生 thinkingConfig，OpenAI-compatible 路径仅按 provider/channel /// 下发官方渠道级字段；OpenAI 官方渠道会跳过普通 chat 模型不支持的字段。详见 issue #402。 @@ -741,10 +780,7 @@ pub struct UserPreferences { pub windows_sendinput_insertion_only: bool, /// Windows：SendInput 模式下是否在系统键盘列表（Win+Space）中显示 OpenLess TSF 输入法。 /// 默认 true 保持现有行为；关闭后用户级禁用语言配置文件，无需管理员权限。 - #[serde( - default = "default_true", - rename = "windowsShowOpenlessInKeyboardList" - )] + #[serde(default = "default_true", rename = "windowsShowOpenlessInKeyboardList")] pub windows_show_openless_in_keyboard_list: bool, /// 用户的工作语言（多选，原生名）。会作为前提注入 LLM polish/translate 的 system prompt 头部， /// 让模型知道该用户在哪些语言间工作。详见 issue #4。 @@ -1069,6 +1105,12 @@ struct UserPreferencesWire { microphone_device_name: String, active_asr_provider: String, active_llm_provider: String, + #[serde(default = "default_pipeline_mode")] + pipeline_mode: PipelineMode, + #[serde(default = "default_multimodal_pipeline_enabled")] + multimodal_pipeline_enabled: bool, + #[serde(default = "default_active_omni_provider")] + active_omni_provider: String, #[serde(default)] llm_thinking_enabled: bool, #[serde(default = "default_true")] @@ -1237,6 +1279,9 @@ impl Default for UserPreferencesWire { microphone_device_name: prefs.microphone_device_name, active_asr_provider: prefs.active_asr_provider, active_llm_provider: prefs.active_llm_provider, + pipeline_mode: prefs.pipeline_mode, + multimodal_pipeline_enabled: prefs.multimodal_pipeline_enabled, + active_omni_provider: prefs.active_omni_provider, llm_thinking_enabled: prefs.llm_thinking_enabled, use_system_proxy: prefs.use_system_proxy, restore_clipboard_after_paste: prefs.restore_clipboard_after_paste, @@ -1333,9 +1378,8 @@ impl<'de> Deserialize<'de> for UserPreferences { // 设置保存都会被热键冲突校验整体拒绝，改动全部丢失（#904）。 let legacy_default_user = cfg!(target_os = "windows") && is_right_control_modifier_shortcut(&dictation_hotkey); - let default_taken_by_dictation = selection_polish_hotkey - .as_ref() - .is_some_and(|binding| { + let default_taken_by_dictation = + selection_polish_hotkey.as_ref().is_some_and(|binding| { crate::shortcut_binding::bindings_overlap(binding, &dictation_hotkey) }); if legacy_default_user || default_taken_by_dictation { @@ -1372,6 +1416,9 @@ impl<'de> Deserialize<'de> for UserPreferences { microphone_device_name: wire.microphone_device_name, active_asr_provider: wire.active_asr_provider, active_llm_provider: wire.active_llm_provider, + pipeline_mode: wire.pipeline_mode, + multimodal_pipeline_enabled: wire.multimodal_pipeline_enabled, + active_omni_provider: wire.active_omni_provider, llm_thinking_enabled: wire.llm_thinking_enabled, use_system_proxy: wire.use_system_proxy, restore_clipboard_after_paste: wire.restore_clipboard_after_paste, @@ -2193,6 +2240,9 @@ impl Default for UserPreferences { microphone_device_name: String::new(), active_asr_provider: default_active_asr_provider(), active_llm_provider: "ark".into(), + pipeline_mode: PipelineMode::Traditional, + multimodal_pipeline_enabled: false, + active_omni_provider: "custom".into(), llm_thinking_enabled: false, use_system_proxy: true, restore_clipboard_after_paste: true, @@ -2954,8 +3004,13 @@ pub struct CapsulePayload { pub struct CredentialsStatus { pub active_asr_provider: String, pub active_llm_provider: String, + /// 当前识别管线模式（"traditional" | "multimodal"），前端据此决定 + /// 配置页渲染哪套卡片、概览页按哪套判定「已配置」。 + pub pipeline_mode: PipelineMode, pub asr_configured: bool, pub llm_configured: bool, + /// 多模态（omni）模型是否已配置。仅 `pipeline_mode == multimodal` 时有意义。 + pub omni_configured: bool, // 兼容旧前端字段（逐步迁移中） pub volcengine_configured: bool, pub ark_configured: bool, @@ -3103,7 +3158,8 @@ mod tests { #[cfg(target_os = "windows")] #[test] - fn new_preferences_keep_the_existing_dictation_default_and_use_right_alt_for_selection_polish() { + fn new_preferences_keep_the_existing_dictation_default_and_use_right_alt_for_selection_polish() + { let prefs = UserPreferences::default(); assert_eq!(prefs.dictation_hotkey.primary, "RightControl"); assert_eq!( @@ -3131,7 +3187,10 @@ mod tests { let prefs: UserPreferences = serde_json::from_str(r#"{"windowsSendInputInsertionOnly": true}"#).unwrap(); assert!(prefs.windows_sendinput_insertion_only); - assert_eq!(prefs.windows_insertion_mode, WindowsInsertionMode::SendInput); + assert_eq!( + prefs.windows_insertion_mode, + WindowsInsertionMode::SendInput + ); } #[test] @@ -3139,7 +3198,10 @@ mod tests { let prefs: UserPreferences = serde_json::from_str(r#"{"windowsSendinputInsertionOnly": true}"#).unwrap(); assert!(prefs.windows_sendinput_insertion_only); - assert_eq!(prefs.windows_insertion_mode, WindowsInsertionMode::SendInput); + assert_eq!( + prefs.windows_insertion_mode, + WindowsInsertionMode::SendInput + ); } #[test] @@ -3205,7 +3267,10 @@ mod tests { assert!(json.contains(r#""windowsInsertionMode":"sendInput""#)); let restored: UserPreferences = serde_json::from_str(&json).unwrap(); assert!(restored.windows_sendinput_insertion_only); - assert_eq!(restored.windows_insertion_mode, WindowsInsertionMode::SendInput); + assert_eq!( + restored.windows_insertion_mode, + WindowsInsertionMode::SendInput + ); } #[test] @@ -3652,6 +3717,7 @@ mod tests { asr_model: Some("fun-asr-realtime".into()), llm_provider: Some("ark".into()), llm_model: Some("deepseek-v3-2".into()), + pipeline_mode: None, asr_ms: Some(230), polish_ms: Some(1450), }; diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 7a658cf05..a3f6c0758 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -377,6 +377,7 @@ export const en: typeof zhCN = { selectHint: 'Select an entry on the left to see details.', recorded: 'Recorded {{duration}}', stepAsr: 'Transcribe', + multimodalPipeline: 'Multimodal', stepAsrHint: 'Time spent waiting for the transcript after key release. Streaming ASR transcribes while you speak, so this is usually much shorter than the recording.', stepPolish: 'Polish', stepInsert: 'Insert', @@ -847,6 +848,13 @@ export const en: typeof zhCN = { asrProviderDesc: 'Switching providers automatically loads the matching credentials.', asrTitle: 'ASR (transcription)', asrDesc: 'Used to turn recorded speech into text.', + omniTitle: 'Multimodal model', + omniDesc: 'One model that turns audio + prompt into the final text directly (experimental pipeline).', + pipelineModeLabel: 'Pipeline mode', + pipelineModeHint: 'Traditional = ASR + LLM two-stage. Multimodal = a single audio-capable model in one pass.', + pipelineModeTraditional: 'Traditional', + pipelineModeMultimodal: 'Multimodal', + pipelineIsolationNotice: 'The two modes keep fully separate credentials. Switching modes keeps the other set stored but unused; switching back restores it.', presets: { ark: 'ARK (Volcengine Ark)', deepseek: 'DeepSeek', @@ -882,6 +890,9 @@ export const en: typeof zhCN = { asrFoundryLocalWhisper: 'Local Whisper (Foundry Local)', asrLocalQwen3: 'Local Qwen3-ASR', asrAppleSpeech: 'Apple Speech (macOS)', + omniOpenai: 'OpenAI (audio-capable)', + omniGemini: 'Google Gemini', + omniDashscope: 'Alibaba DashScope Omni', }, elevenLabsUploadNotice: 'ElevenLabs uploads recorded audio to the configured endpoint for batch transcription.', zenmuxVocabularyNote: 'ZenMux uses a JSON transcription protocol and does not receive dictionary hotwords (prompt/hotwords); the dictionary still feeds the polish step but does not bias speech recognition.', @@ -1084,6 +1095,10 @@ export const en: typeof zhCN = { }, }, advanced: { + multimodalPipelineTitle: 'Multimodal recognition pipeline (experimental)', + multimodalPipelineTitleHint: 'One-pass audio recognition with a single multimodal model; traditional ASR + LLM configuration is fully isolated from it.', + multimodalPipelineLabel: 'Enable multimodal pipeline', + multimodalPipelineHint: 'Adds a Traditional / Multimodal switch on the AI providers page. Traditional = ASR + LLM; Multimodal = one audio-capable model. The two configurations are stored separately and never share credentials.', streamingInsertTitle: 'Streaming insertion', streamingInsertTitleLinux: 'Streaming insertion (experimental)', streamingInsertDesc: diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 013b16e7e..15905a498 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -379,6 +379,7 @@ export const ja: typeof zhCN = { selectHint: '左側から 1 件選択して詳細を表示。', recorded: '録音 {{duration}}', stepAsr: '認識', + multimodalPipeline: 'マルチモーダル', stepAsrHint: 'キーを離してから認識結果を待った時間。ストリーミング認識は録音中に変換するため、通常は録音時間よりずっと短くなります。', stepPolish: '推敲', stepInsert: '挿入', @@ -849,6 +850,13 @@ export const ja: typeof zhCN = { asrProviderDesc: '切り替えると対応する認証情報が自動選択されます。', asrTitle: 'ASR 音声（転写）', asrDesc: '録音した音声をテキストに文字起こしします。', + omniTitle: 'マルチモーダルモデル', + omniDesc: '1つのモデルが「プロンプト + 音声」から最終テキストを直接出力します（実験的パイプライン）。', + pipelineModeLabel: '認識パイプライン', + pipelineModeHint: '従来 = ASR 文字起こし + LLM 整形の2段式。マルチモーダル = 音声対応モデルが1回で完了。', + pipelineModeTraditional: '従来モード', + pipelineModeMultimodal: 'マルチモーダルモード', + pipelineIsolationNotice: '2つのモードは完全に独立した認証情報を使用します。切り替えてももう一方の設定は削除されず、切り戻せば復元されます。', presets: { ark: 'ARK（Volcengine Ark）', deepseek: 'DeepSeek', @@ -884,6 +892,9 @@ export const ja: typeof zhCN = { asrFoundryLocalWhisper: 'ローカル Whisper（Foundry Local）', asrLocalQwen3: 'ローカル Qwen3-ASR', asrAppleSpeech: 'Apple 音声認識 (macOS)', + omniOpenai: 'OpenAI（音声対応）', + omniGemini: 'Google Gemini', + omniDashscope: 'Alibaba DashScope Omni', }, elevenLabsUploadNotice: 'ElevenLabs は録音音声を設定済みのエンドポイントへアップロードしてバッチ文字起こしします。', zenmuxVocabularyNote: 'ZenMux は JSON 文字起こしプロトコルを使用し、辞書ホットワード（prompt/hotwords）は送信されません。辞書は依然として潤色段階には渡りますが、音声認識のバイアスには使用されません。', @@ -1052,6 +1063,10 @@ export const ja: typeof zhCN = { }, }, advanced: { + multimodalPipelineTitle: 'マルチモーダル認識パイプライン（実験的）', + multimodalPipelineTitleHint: '1つのマルチモーダルモデルで音声認識を一括実行。従来の ASR + LLM 設定から完全に分離されます。', + multimodalPipelineLabel: 'マルチモーダルパイプラインを有効化', + multimodalPipelineHint: '有効にすると「サービス → AI プロバイダー」ページに従来 / マルチモーダルの切り替えが表示されます。従来 = ASR + LLM、マルチモーダル = 音声対応モデル1つ。設定は別々に保存され、認証情報を共有しません。', streamingInsertTitle: 'ストリーミング入力', streamingInsertTitleLinux: 'ストリーミング入力（実験的）', streamingInsertDesc: diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index ed2d5a591..19c70a398 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -379,6 +379,7 @@ export const ko: typeof zhCN = { selectHint: '왼쪽에서 하나를 선택하여 자세히 보기.', recorded: '녹음 {{duration}}', stepAsr: '인식', + multimodalPipeline: '멀티모달', stepAsrHint: '키를 뗀 후 인식 결과를 기다린 시간. 스트리밍 인식은 녹음 중에 변환하므로 보통 녹음 시간보다 훨씬 짧습니다.', stepPolish: '다듬기', stepInsert: '삽입', @@ -849,6 +850,13 @@ export const ko: typeof zhCN = { asrProviderDesc: '전환 시 해당하는 자격 증명이 자동 선택됩니다.', asrTitle: 'ASR 음성(전사)', asrDesc: '녹음된 음성을 텍스트로 전사합니다.', + omniTitle: '멀티모달 모델', + omniDesc: '하나의 모델이 프롬프트 + 오디오를 받아 최종 텍스트를 한 번에 출력합니다(실험적 파이프라인).', + pipelineModeLabel: '인식 파이프라인', + pipelineModeHint: '전통 = ASR 전사 + LLM 다듬기 2단계. 멀티모달 = 오디오 지원 모델이 한 번에 처리.', + pipelineModeTraditional: '전통 모드', + pipelineModeMultimodal: '멀티모달 모드', + pipelineIsolationNotice: '두 모드는 완전히 분리된 자격 증명을 사용합니다. 전환해도 다른 쪽 설정은 삭제되지 않으며, 다시 전환하면 복원됩니다.', presets: { ark: 'ARK (Volcengine Ark)', deepseek: 'DeepSeek', @@ -884,6 +892,9 @@ export const ko: typeof zhCN = { asrFoundryLocalWhisper: '로컬 Whisper(Foundry Local)', asrLocalQwen3: '로컬 Qwen3-ASR', asrAppleSpeech: 'Apple 음성 (macOS)', + omniOpenai: 'OpenAI (오디오 지원)', + omniGemini: 'Google Gemini', + omniDashscope: 'Alibaba DashScope Omni', }, elevenLabsUploadNotice: 'ElevenLabs는 녹음 오디오를 설정된 엔드포인트에 업로드해 일괄 전사합니다.', zenmuxVocabularyNote: 'ZenMux는 JSON 전사 프로토콜을 사용하며 사전 핫워드(prompt/hotwords)를 보내지 않습니다. 사전은 여전히 다듬기 단계에 전달되지만 음성 인식 편향에는 사용되지 않습니다.', @@ -1052,6 +1063,10 @@ export const ko: typeof zhCN = { }, }, advanced: { + multimodalPipelineTitle: '멀티모달 인식 파이프라인 (실험적)', + multimodalPipelineTitleHint: '단일 멀티모달 모델로 음성 인식을 한 번에 처리합니다. 기존 ASR + LLM 설정과 완전히 분리됩니다.', + multimodalPipelineLabel: '멀티모달 파이프라인 활성화', + multimodalPipelineHint: '활성화하면 「서비스 → AI 공급자」 페이지에 전통 / 멀티모달 전환이 나타납니다. 전통 = ASR + LLM, 멀티모달 = 오디오 지원 모델 1개. 두 설정은 별도로 저장되며 자격 증명을 공유하지 않습니다.', streamingInsertTitle: '스트리밍 입력', streamingInsertTitleLinux: '스트리밍 입력 (실험적)', streamingInsertDesc: diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 77574093f..f75b040f1 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -375,6 +375,7 @@ export const zhCN = { selectHint: '左侧选一条查看详情。', recorded: '录音 {{duration}}', stepAsr: '识别', + multimodalPipeline: '多模态', stepAsrHint: '松键后等待识别结果的耗时。流式识别边录边转，此值通常远小于录音时长。', stepPolish: '润色', stepInsert: '插入', @@ -845,6 +846,13 @@ export const zhCN = { asrProviderDesc: '切换后将自动选用对应凭据。', asrTitle: 'ASR 语音（转写）', asrDesc: '用于将录制的语音转写为文本。', + omniTitle: '多模态模型', + omniDesc: '一个模型直接接收「提示词 + 音频」一步输出最终文本（实验性管线）。', + pipelineModeLabel: '识别管线', + pipelineModeHint: '传统 = ASR 转写 + LLM 润色两段式；多模态 = 单个多模态模型一次完成。', + pipelineModeTraditional: '传统模式', + pipelineModeMultimodal: '多模态模式', + pipelineIsolationNotice: '两种模式使用完全独立的凭据配置。切换模式不会删除另一套配置，只是暂时停用；切回即恢复。', presets: { ark: 'ARK（火山方舟）', deepseek: 'DeepSeek', @@ -880,6 +888,9 @@ export const zhCN = { asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）', asrLocalQwen3: '本地 Qwen3-ASR', asrAppleSpeech: 'Apple 语音（macOS）', + omniOpenai: 'OpenAI（支持音频）', + omniGemini: 'Google Gemini', + omniDashscope: '阿里云百炼 Omni', }, elevenLabsUploadNotice: 'ElevenLabs 会将录音上传到所配置的端点进行批量转写。', zenmuxVocabularyNote: 'ZenMux 走 JSON 转写协议，不发送词典热词（prompt/hotwords）；词典仍会进入润色链路，但不会参与语音识别偏置。', @@ -1082,6 +1093,10 @@ export const zhCN = { }, }, advanced: { + multimodalPipelineTitle: '多模态识别管线（实验性）', + multimodalPipelineTitleHint: '用单个多模态模型一步完成语音识别；与传统 ASR + LLM 配置完全隔离。', + multimodalPipelineLabel: '启用多模态识别管线', + multimodalPipelineHint: '开启后，「服务 → AI 提供商」页出现「传统模式 / 多模态模式」切换。传统 = ASR + LLM；多模态 = 单个支持音频的模型。两套配置分开存储、绝不共享凭据。', streamingInsertTitle: '流式输入', streamingInsertTitleLinux: '流式输入（实验）', streamingInsertDesc: diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 846e1fe62..ad2177274 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -377,6 +377,7 @@ export const zhTW: typeof zhCN = { selectHint: '左側選一條查看詳情。', recorded: '錄音 {{duration}}', stepAsr: '辨識', + multimodalPipeline: '多模態', stepAsrHint: '放開按鍵後等待辨識結果的耗時。串流辨識邊錄邊轉，此值通常遠小於錄音時長。', stepPolish: '潤飾', stepInsert: '插入', @@ -847,6 +848,13 @@ export const zhTW: typeof zhCN = { asrProviderDesc: '切換後將自動選用對應憑據。', asrTitle: 'ASR 語音（轉寫）', asrDesc: '用於將錄製的語音轉寫為文字。', + omniTitle: '多模態模型', + omniDesc: '一個模型直接接收「提示詞 + 音訊」一步輸出最終文字（實驗性管線）。', + pipelineModeLabel: '識別管線', + pipelineModeHint: '傳統 = ASR 轉寫 + LLM 潤色兩段式；多模態 = 單一多模態模型一次完成。', + pipelineModeTraditional: '傳統模式', + pipelineModeMultimodal: '多模態模式', + pipelineIsolationNotice: '兩種模式使用完全獨立的憑證設定。切換模式不會刪除另一套設定，只是暫時停用；切回即恢復。', presets: { ark: 'ARK（火山方舟）', deepseek: 'DeepSeek', @@ -882,6 +890,9 @@ export const zhTW: typeof zhCN = { asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）', asrLocalQwen3: '本地 Qwen3-ASR', asrAppleSpeech: 'Apple 語音（macOS）', + omniOpenai: 'OpenAI（支援音訊）', + omniGemini: 'Google Gemini', + omniDashscope: '阿里雲百煉 Omni', }, elevenLabsUploadNotice: 'ElevenLabs 會將錄音上傳至已設定的端點進行批次轉寫。', zenmuxVocabularyNote: 'ZenMux 走 JSON 轉寫協定，不傳送詞典熱詞（prompt/hotwords）；詞典仍會進入潤色鏈路，但不會參與語音辨識偏置。', @@ -1050,6 +1061,10 @@ export const zhTW: typeof zhCN = { }, }, advanced: { + multimodalPipelineTitle: '多模態辨識管線（實驗性）', + multimodalPipelineTitleHint: '用單一多模態模型一步完成語音辨識；與傳統 ASR + LLM 設定完全隔離。', + multimodalPipelineLabel: '啟用多模態辨識管線', + multimodalPipelineHint: '開啟後，「服務 → AI 提供者」頁出現「傳統模式 / 多模態模式」切換。傳統 = ASR + LLM；多模態 = 單一支援音訊的模型。兩套設定分開儲存、絕不共用憑證。', streamingInsertTitle: '流式輸入', streamingInsertTitleLinux: '流式輸入（實驗）', streamingInsertDesc: diff --git a/openless-all/app/src/lib/ipc/asr-credentials.ts b/openless-all/app/src/lib/ipc/asr-credentials.ts index 49ebe0ca9..25e7e3c28 100644 --- a/openless-all/app/src/lib/ipc/asr-credentials.ts +++ b/openless-all/app/src/lib/ipc/asr-credentials.ts @@ -1,66 +1,74 @@ -import type { CredentialsStatus } from "../types" -import { invokeOrMock } from "./shared" -import { mockCredentialsStatus } from "./mock-data" - -export interface ProviderCheckResult { - ok: boolean -} - -export interface ProviderModelsResult { - models: string[] -} - -export function getCredentials(): Promise { - return invokeOrMock( - "get_credentials", - undefined, - () => mockCredentialsStatus, - ) -} - -export function setCredential(account: string, value: string, provider?: string): Promise { - return invokeOrMock("set_credential", { account, value, provider }, () => undefined) -} - -export function setActiveAsrProvider(provider: string): Promise { +import type { CredentialsStatus } from "../types" +import { invokeOrMock } from "./shared" +import { mockCredentialsStatus } from "./mock-data" + +export interface ProviderCheckResult { + ok: boolean +} + +export interface ProviderModelsResult { + models: string[] +} + +export function getCredentials(): Promise { + return invokeOrMock( + "get_credentials", + undefined, + () => mockCredentialsStatus, + ) +} + +export function setCredential(account: string, value: string, provider?: string): Promise { + return invokeOrMock("set_credential", { account, value, provider }, () => undefined) +} + +export function setActiveAsrProvider(provider: string): Promise { + return invokeOrMock( + "set_active_asr_provider", + { provider }, + () => undefined, + ) +} + +export function setActiveLlmProvider(provider: string): Promise { return invokeOrMock( - "set_active_asr_provider", + "set_active_llm_provider", { provider }, () => undefined, ) } -export function setActiveLlmProvider(provider: string): Promise { +export function setActiveOmniProvider(provider: string): Promise { return invokeOrMock( - "set_active_llm_provider", + "set_active_omni_provider", { provider }, () => undefined, ) } -export function readCredential(account: string, provider?: string): Promise { - return invokeOrMock( - "read_credential", - { account, provider }, - () => null, - ) -} - +export function readCredential(account: string, provider?: string): Promise { + return invokeOrMock( + "read_credential", + { account, provider }, + () => null, + ) +} + export function validateProviderCredentials( - kind: "llm" | "asr", + kind: "llm" | "asr" | "omni", ): Promise { - return invokeOrMock("validate_provider_credentials", { kind }, () => ({ - ok: true, - })) -} - + return invokeOrMock("validate_provider_credentials", { kind }, () => ({ + ok: true, + })) +} + export function listProviderModels( - kind: "llm" | "asr", + kind: "llm" | "asr" | "omni", ): Promise { - return invokeOrMock("list_provider_models", { kind }, () => ({ - models: - kind === "llm" - ? ["gpt-4o", "deepseek-v4-flash", "deepseek-v4-pro"] - : ["whisper-1"], - })) -} + return invokeOrMock("list_provider_models", { kind }, () => ({ + models: + kind === "llm" + ? ["gpt-4o", "deepseek-v4-flash", "deepseek-v4-pro"] + : ["whisper-1"], + })) +} diff --git a/openless-all/app/src/lib/ipc/index.ts b/openless-all/app/src/lib/ipc/index.ts index b92aa5683..8b0f13dd1 100644 --- a/openless-all/app/src/lib/ipc/index.ts +++ b/openless-all/app/src/lib/ipc/index.ts @@ -27,6 +27,7 @@ export { setCredential, setActiveAsrProvider, setActiveLlmProvider, + setActiveOmniProvider, readCredential, validateProviderCredentials, listProviderModels, diff --git a/openless-all/app/src/lib/ipc/mock-data.ts b/openless-all/app/src/lib/ipc/mock-data.ts index 9a0645c15..c3590fddd 100644 --- a/openless-all/app/src/lib/ipc/mock-data.ts +++ b/openless-all/app/src/lib/ipc/mock-data.ts @@ -50,6 +50,9 @@ export let mockSettings: UserPreferences = { microphoneDeviceName: "", activeAsrProvider: "foundry-local-whisper", activeLlmProvider: "ark", + pipelineMode: "traditional", + multimodalPipelineEnabled: false, + activeOmniProvider: "custom", llmThinkingEnabled: false, useSystemProxy: true, restoreClipboardAfterPaste: true, @@ -534,8 +537,10 @@ export const mockHotkeyCapability: HotkeyCapability = { export const mockCredentialsStatus: CredentialsStatus = { activeAsrProvider: "foundry-local-whisper", activeLlmProvider: "ark", + pipelineMode: "traditional", asrConfigured: true, llmConfigured: true, + omniConfigured: false, volcengineConfigured: true, arkConfigured: true, } diff --git a/openless-all/app/src/lib/providerSetup.test.ts b/openless-all/app/src/lib/providerSetup.test.ts index 06622a6bb..a6b657faf 100644 --- a/openless-all/app/src/lib/providerSetup.test.ts +++ b/openless-all/app/src/lib/providerSetup.test.ts @@ -1,110 +1,154 @@ -import { - areProvidersConfigured, - shouldShowProviderSetupPrompt, -} from './providerSetup'; - -function assertEqual(actual: boolean, expected: boolean, name: string) { - if (actual !== expected) { - throw new Error(`${name}: expected ${expected}, got ${actual}`); - } -} - -assertEqual( +import { + areProvidersConfigured, + shouldShowProviderSetupPrompt, +} from './providerSetup'; + +function assertEqual(actual: boolean, expected: boolean, name: string) { + if (actual !== expected) { + throw new Error(`${name}: expected ${expected}, got ${actual}`); + } +} + +assertEqual( areProvidersConfigured({ activeAsrProvider: 'volcengine', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: true, llmConfigured: true, + omniConfigured: false, volcengineConfigured: true, arkConfigured: true, - }), - true, - 'configured when ASR and LLM are both ready', -); - -assertEqual( + }), + true, + 'configured when ASR and LLM are both ready', +); + +assertEqual( areProvidersConfigured({ activeAsrProvider: 'volcengine', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: false, llmConfigured: true, + omniConfigured: false, volcengineConfigured: false, arkConfigured: true, - }), - false, - 'not configured when ASR provider is missing', -); - -assertEqual( + }), + false, + 'not configured when ASR provider is missing', +); + +assertEqual( areProvidersConfigured({ activeAsrProvider: 'volcengine', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: true, llmConfigured: false, + omniConfigured: false, volcengineConfigured: true, arkConfigured: false, - }), - false, - 'not configured when LLM provider is missing', -); - -assertEqual( + }), + false, + 'not configured when LLM provider is missing', +); + +assertEqual( areProvidersConfigured({ activeAsrProvider: 'whisper', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: true, llmConfigured: true, + omniConfigured: false, volcengineConfigured: false, arkConfigured: true, - }), - true, - 'configured when active ASR is non-volcengine but already ready', -); - -assertEqual( - shouldShowProviderSetupPrompt( - { + }), + true, + 'configured when active ASR is non-volcengine but already ready', +); + +assertEqual( + shouldShowProviderSetupPrompt( + { activeAsrProvider: 'whisper', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: false, llmConfigured: false, + omniConfigured: false, volcengineConfigured: false, arkConfigured: false, - }, - null, - ), - true, - 'show first-run prompt when providers are missing and no prompt was seen', -); - -assertEqual( - shouldShowProviderSetupPrompt( - { + }, + null, + ), + true, + 'show first-run prompt when providers are missing and no prompt was seen', +); + +assertEqual( + shouldShowProviderSetupPrompt( + { activeAsrProvider: 'whisper', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: false, llmConfigured: false, + omniConfigured: false, volcengineConfigured: false, arkConfigured: false, - }, - '1', - ), - false, - 'do not repeat first-run prompt after the user has deferred it in this session', -); - -assertEqual( - shouldShowProviderSetupPrompt( - { + }, + '1', + ), + false, + 'do not repeat first-run prompt after the user has deferred it in this session', +); + +assertEqual( + shouldShowProviderSetupPrompt( + { activeAsrProvider: 'whisper', activeLlmProvider: 'ark', + pipelineMode: 'traditional', asrConfigured: true, llmConfigured: true, + omniConfigured: false, volcengineConfigured: false, arkConfigured: true, - }, - null, - ), - false, + }, + null, + ), + false, 'do not show prompt when providers are already configured', ); + +assertEqual( + areProvidersConfigured({ + activeAsrProvider: 'volcengine', + activeLlmProvider: 'ark', + pipelineMode: 'multimodal', + asrConfigured: false, + llmConfigured: false, + omniConfigured: true, + volcengineConfigured: false, + arkConfigured: false, + }), + true, + 'multimodal mode only requires the omni model', +); + +assertEqual( + areProvidersConfigured({ + activeAsrProvider: 'volcengine', + activeLlmProvider: 'ark', + pipelineMode: 'multimodal', + asrConfigured: true, + llmConfigured: true, + omniConfigured: false, + volcengineConfigured: true, + arkConfigured: true, + }), + false, + 'multimodal mode ignores traditional ASR/LLM readiness', +); diff --git a/openless-all/app/src/lib/providerSetup.ts b/openless-all/app/src/lib/providerSetup.ts index fcb624c95..f82b362a6 100644 --- a/openless-all/app/src/lib/providerSetup.ts +++ b/openless-all/app/src/lib/providerSetup.ts @@ -1,16 +1,20 @@ -import type { CredentialsStatus } from './types'; - -export const PROVIDER_SETUP_PROMPT_DEFERRED_KEY = 'ol.providerSetupPromptDeferredThisSession'; - +import type { CredentialsStatus } from './types'; + +export const PROVIDER_SETUP_PROMPT_DEFERRED_KEY = 'ol.providerSetupPromptDeferredThisSession'; + export function areProvidersConfigured(credentials: CredentialsStatus): boolean { + // 多模态（Omni）模式：只要求多模态模型已配置；传统 ASR/LLM 两套在该模式下不参与。 + if (credentials.pipelineMode === 'multimodal') { + return credentials.omniConfigured === true; + } const asrConfigured = credentials.asrConfigured ?? credentials.volcengineConfigured; const llmConfigured = credentials.llmConfigured ?? credentials.arkConfigured; return asrConfigured && llmConfigured; } - -export function shouldShowProviderSetupPrompt( - credentials: CredentialsStatus, - promptDeferredValue: string | null, -): boolean { - return !areProvidersConfigured(credentials) && promptDeferredValue !== '1'; -} + +export function shouldShowProviderSetupPrompt( + credentials: CredentialsStatus, + promptDeferredValue: string | null, +): boolean { + return !areProvidersConfigured(credentials) && promptDeferredValue !== '1'; +} diff --git a/openless-all/app/src/lib/stylePrefs.test.ts b/openless-all/app/src/lib/stylePrefs.test.ts index 1d84aac65..e63cd7a70 100644 --- a/openless-all/app/src/lib/stylePrefs.test.ts +++ b/openless-all/app/src/lib/stylePrefs.test.ts @@ -18,6 +18,9 @@ function assert(condition: boolean, message: string) { const previousPrefs: UserPreferences = { hotkey: { trigger: 'rightOption', mode: 'toggle' }, dictationHotkey: { primary: 'RightOption', modifiers: [] }, + pipelineMode: 'traditional', + multimodalPipelineEnabled: false, + activeOmniProvider: 'custom', selectionPolishHotkey: { primary: 'RightControl', modifiers: [] }, selectionPolishStylePackId: 'builtin.light', selectionPolishOutputMode: 'directReplace', diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index 9af16fbcf..9c21df090 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -24,6 +24,11 @@ export type { export type PolishMode = 'raw' | 'light' | 'structured' | 'formal'; +/** 识别管线模式（issue #902）：traditional = ASR + LLM 两段式； + * multimodal = 单个多模态模型一步完成「音频 + 提示词 → 最终文本」。 + * 两套配置在凭据库中完全隔离，运行时只读当前模式。 */ +export type PipelineMode = 'traditional' | 'multimodal'; + export type InsertStatus = 'inserted' | 'pasteSent' | 'copiedFallback' | 'failed'; /** 概览页年度活动热力图的单日计数（date = 本地日期 YYYY-MM-DD）。 */ @@ -58,6 +63,8 @@ export interface DictationSession { llmProvider: string | null; /** 本次润色用的 LLM 模型 id。Raw 直通时为 null。 */ llmModel: string | null; + /** 本次会话走的识别管线模式（"multimodal" / 缺失 = 传统两段式）。 */ + pipelineMode?: string | null; /** 松键后等待转写结果的实测耗时（毫秒）。流式 ASR 是收尾延迟，批式是完整转写耗时。 */ asrMs: number | null; /** LLM 润色/翻译调用的实测耗时（毫秒）。未调用 LLM 时为 null。 */ @@ -297,6 +304,12 @@ export interface UserPreferences { microphoneDeviceName: string; activeAsrProvider: string; activeLlmProvider: string; + /** 识别管线模式（实验性，issue #902）。multimodal 时各语音管线改用 omni 配置。 */ + pipelineMode: PipelineMode; + /** 「多模态识别管线」实验性功能总开关（高级设置）。默认 false。 */ + multimodalPipelineEnabled: boolean; + /** 多模态（Omni）模型当前激活的 provider id，镜像凭据库 omni.active。 */ + activeOmniProvider: string; /** LLM 思考模式开关。默认关闭；OpenAI 普通 chat 模型会跳过不支持的字段。详见 issue #402。 */ llmThinkingEnabled: boolean; /** 是否使用系统代理（issue #869）。默认开启；关闭后所有请求直连，境外服务（GitHub 登录/更新等）可能连不上。 */ @@ -604,8 +617,12 @@ export interface CapsulePayload { export interface CredentialsStatus { activeAsrProvider: string; activeLlmProvider: string; + /** 当前识别管线模式，前端据此渲染配置页与概览「已配置」判定。 */ + pipelineMode: PipelineMode; asrConfigured: boolean; llmConfigured: boolean; + /** 多模态（omni）模型是否已配置。仅 multimodal 模式有意义。 */ + omniConfigured: boolean; /** 兼容旧字段（过渡期保留）。 */ volcengineConfigured: boolean; arkConfigured: boolean; diff --git a/openless-all/app/src/pages/History.tsx b/openless-all/app/src/pages/History.tsx index 2bfa18c95..4c9bf57e9 100644 --- a/openless-all/app/src/pages/History.tsx +++ b/openless-all/app/src/pages/History.tsx @@ -1,655 +1,659 @@ -// History.tsx — 接 Tauri 后端 list_history / delete_history_entry / clear_history。 -// 真实数据来自 ~/Library/Application Support/OpenLess/history.json。 - -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { useTranslation } from 'react-i18next'; -import { Icon } from '../components/Icon'; -import { Tooltip } from '../components/Tooltip'; -import { detectOS } from '../components/WindowChrome'; -import { formatComboLabel } from '../lib/hotkey'; -import { clearHistory, deleteHistoryEntry, listHistory, readAudioRecording, retranscribeRecording, isTauri } from '../lib/ipc'; -import { useMobileLayout } from '../lib/useMobileLayout'; -import type { DictationSession, PolishMode } from '../lib/types'; -import { useHotkeySettings } from '../state/HotkeySettingsContext'; -import { Btn, Card, PageHeader, Pill } from './_atoms'; -import { chipSelectedStyle } from './settings/shared'; - -function useFilters(): Array<{ id: 'all' | PolishMode; label: string }> { - const { t } = useTranslation(); - return [ - { id: 'all', label: t('history.filterAll') }, - { id: 'raw', label: t('style.modes.raw.name') }, - { id: 'light', label: t('style.modes.light.name') }, - { id: 'structured', label: t('style.modes.structured.name') }, - { id: 'formal', label: t('style.modes.formal.name') }, - ]; -} - -function useModeLabel(): Record { - const { t } = useTranslation(); - return { - raw: t('style.modes.raw.name'), - light: t('style.modes.light.name'), - structured: t('style.modes.structured.name'), - formal: t('style.modes.formal.name'), - }; -} - -export function History() { - const { t } = useTranslation(); - const os = detectOS(); - const FILTERS = useFilters(); - const MODE_LABEL = useModeLabel(); - const [filter, setFilter] = useState<'all' | PolishMode>('all'); - const [query, setQuery] = useState(''); - const [debouncedQuery, setDebouncedQuery] = useState(''); - const [items, setItems] = useState([]); - const [selectedId, setSelectedId] = useState(null); - const [loading, setLoading] = useState(true); - const [loadError, setLoadError] = useState(null); - const [actionError, setActionError] = useState(null); - const [justCopied, setJustCopied] = useState(false); - const [justCopiedRaw, setJustCopiedRaw] = useState(false); - // 「重新转录」进行中：禁用按钮 + 显示「转录中…」，避免重复点击发起多次 ASR。 - const [retranscribing, setRetranscribing] = useState(false); - // 录音文件 lazily-detected missing 状态：retention / 条数 cap 清理后磁盘上 wav - // 可能已被删，但 history 条目 hasAudioRecording 仍写 true。任一组件 - // （播放 / 导出）首次 IPC 拿到 'recording not found' 时把 id 加进来， - // 之后渲染按钮的条件就转 false，避免反复点击得到同样的 error。 - // 修 pr_agent "Missing file check" 反馈。 - const [audioMissingIds, setAudioMissingIds] = useState>(() => new Set()); - const markAudioMissing = useCallback((id: string) => { - setAudioMissingIds(prev => { - if (prev.has(id)) return prev; - const next = new Set(prev); - next.add(id); - return next; - }); - }, []); - const { prefs } = useHotkeySettings(); - const mobile = useMobileLayout(); - const [mobileDetailOpen, setMobileDetailOpen] = useState(false); - - const refresh = useCallback(async () => { - setLoading(true); - setLoadError(null); - try { - const data = await listHistory(); - setItems(data); - setActionError(null); - setSelectedId(prev => (prev && data.some(s => s.id === prev) ? prev : data[0]?.id ?? null)); - } catch (error) { - console.error('[history] failed to load history', error); - setLoadError(errorMessage(error)); - } finally { - setLoading(false); - } - }, []); - - useEffect(() => { - void refresh(); - }, [refresh]); - - const searchInputRef = useRef(null); - const searchShortcut = os === 'mac' ? '⌘K' : 'Ctrl+K'; - - // 搜索词防抖：随输入实时更新 query，300ms 后落到 debouncedQuery 再过滤， - // 避免每个按键都重算整张列表（与 Marketplace 同模式）。 - useEffect(() => { - const id = window.setTimeout(() => setDebouncedQuery(query), 300); - return () => window.clearTimeout(id); - }, [query]); - - // ⌘K / Ctrl+K 聚焦搜索框（设计稿提示的快捷键）；⌘R / Ctrl+R 刷新历史列表 - // （与浏览器「重新加载」直觉一致）。preventDefault 拦掉 webview 默认的整页 - // reload，改为只重拉 listHistory，避免整个前端重挂载。 - useEffect(() => { - const onKeyDown = (e: KeyboardEvent) => { - if ((e.metaKey || e.ctrlKey) && (e.key === 'k' || e.key === 'K')) { - e.preventDefault(); - searchInputRef.current?.focus(); - return; - } - if ((e.metaKey || e.ctrlKey) && (e.key === 'r' || e.key === 'R')) { - e.preventDefault(); - void refresh(); - } - }; - window.addEventListener('keydown', onKeyDown); - return () => window.removeEventListener('keydown', onKeyDown); - }, [refresh]); - - const filtered = useMemo(() => { - const byMode = filter === 'all' ? items : items.filter(s => s.mode === filter); - const q = debouncedQuery.trim().toLowerCase(); - if (!q) return byMode; - // 按原始转写 + 润色后文本匹配关键词，覆盖用户能想起的两种内容。 - return byMode.filter( - s => - s.rawTranscript.toLowerCase().includes(q) || - s.finalText.toLowerCase().includes(q), - ); - }, [items, filter, debouncedQuery]); - const item = useMemo( - () => filtered.find(s => s.id === selectedId) || filtered[0], - [filtered, selectedId], - ); - - const onClear = async () => { - if (items.length === 0) return; - if (!confirm(t('history.confirmClear', { count: items.length }))) return; - setActionError(null); - try { - await clearHistory(); - setItems([]); - setSelectedId(null); - } catch (error) { - console.error('[history] failed to clear history', error); - setActionError(t('history.clearFailed', { err: errorMessage(error) })); - } - }; - - const onDelete = async () => { - if (!item) return; - const deletedId = item.id; - setActionError(null); - try { - await deleteHistoryEntry(deletedId); - setItems(prev => prev.filter(s => s.id !== deletedId)); - setSelectedId(current => (current === deletedId ? null : current)); - } catch (error) { - console.error('[history] failed to delete history entry', error); - setActionError(t('history.deleteFailed', { err: errorMessage(error) })); - } - }; - - const onCopy = async () => { - if (!item) return; - try { - if (!navigator.clipboard?.writeText) { - throw new Error('clipboard unavailable'); - } - // 润色失败/未产出时 finalText 为空，回退到原文，避免「复制」按钮复制空字符串 - // 导致原文无法从 UI 取回（polish 失败时仍能拿到识别原文）。 - await navigator.clipboard.writeText(item.finalText.trim() ? item.finalText : item.rawTranscript); - setActionError(null); - setJustCopied(true); - window.setTimeout(() => setJustCopied(false), 1500); - } catch (error) { - console.error('[history] failed to copy entry', error); - setActionError(t('history.copyFailed', { err: errorMessage(error) })); - } - }; - - // 原文（识别结果）单独复制：润色失败或用户只想要未润色文本时使用。 - const onCopyRaw = async () => { - if (!item) return; - try { - if (!navigator.clipboard?.writeText) { - throw new Error('clipboard unavailable'); - } - await navigator.clipboard.writeText(item.rawTranscript); - setActionError(null); - setJustCopiedRaw(true); - window.setTimeout(() => setJustCopiedRaw(false), 1500); - } catch (error) { - console.error('[history] failed to copy raw transcript', error); - setActionError(t('history.copyFailed', { err: errorMessage(error) })); - } - }; - - const onExportAudio = async () => { - if (!item || !item.hasAudioRecording) return; - try { - // Wry/WebKit 中 data URL 的 可能不触发保存对话框，后端直接调系统对话框 - if (isTauri) { - const { invoke } = await import('@tauri-apps/api/core'); - await invoke('export_audio_recording', { sessionId: item.id }); - } else { - const dataUrl = await readAudioRecording(item.id); - if (!dataUrl || dataUrl === 'data:audio/wav;base64,') throw new Error('empty recording'); - const a = document.createElement('a'); - a.href = dataUrl; - a.download = `openless-recording-${item.id}.wav`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - } - setActionError(null); - } catch (error) { - console.error('[history] failed to export recording', error); - const msg = errorMessage(error); - if (isUserCancelled(msg)) { - setActionError(null); - return; - } - if (msg === 'recording export failed') { - setActionError(t('history.exportError')); - return; - } - // wav 已被 retention / 条数 cap 清理：把按钮隐藏，不显示错误（用户没干错事）。 - if (msg.includes('recording not found') || msg.includes('not found')) { - markAudioMissing(item.id); - return; - } - setActionError(t('history.exportFailed', { err: msg })); - } - }; - - // 对一条「转录失败 / 没识别到语音」的历史用当前 ASR provider 重新转录（issue #613）。 - // 后端读 recordings/.wav → 重转 → 原地回写该条 rawTranscript/finalText、清 errorCode， - // 返回整条记录；前端据此局部刷新。失败保留 + 自动重试已让这些条目的录音留得住，这里给 - // 持久失败（重试也没救回来）一个手动重转入口。 - const onRetranscribe = async () => { - if (!item || !item.hasAudioRecording) return; - setRetranscribing(true); - setActionError(null); - try { - const updated = await retranscribeRecording(item.id); - setItems(prev => prev.map(s => (s.id === updated.id ? updated : s))); - } catch (error) { - console.error('[history] retranscribe failed', error); - const msg = errorMessage(error); - // wav 已被 retention / 条数 cap 清理：隐藏入口，不报错（用户没干错事）。 - if (msg.includes('recording not found') || msg.includes('not found')) { - markAudioMissing(item.id); - return; - } - setActionError(t('history.retranscribeFailed', { err: msg })); - } finally { - setRetranscribing(false); - } - }; - - return ( -
- - void refresh()}>{t('common.refresh')} - {t('common.clear')} -
- } - /> -
- {( !mobile || !mobileDetailOpen) && ( - -
-
- - setQuery(e.target.value)} - placeholder={t('history.searchPlaceholder', { shortcut: searchShortcut })} - aria-label={t('history.searchPlaceholder', { shortcut: searchShortcut })} - style={{ - flex: 1, minWidth: 0, - outline: 'none', border: 0, background: 'transparent', - fontSize: 12, color: 'var(--ol-ink-1)', fontFamily: 'inherit', - }} - /> -
-
- {t('history.summary', { total: items.length, shown: filtered.length })} -
-
- {FILTERS.map(f => ( - - ))} -
-
-
- {actionError && ( -
- {actionError} -
- )} - {loading &&
{t('common.loading')}
} - {!loading && loadError && ( -
- {t('history.loadFailed', { err: loadError })} - void refresh()}>{t('history.retry')} -
- )} - {!loading && !loadError && filtered.length === 0 && ( -
- {debouncedQuery.trim() - ? t('history.searchNoMatch', { query: debouncedQuery.trim() }) - : t('history.empty', { trigger: prefs ? formatComboLabel(prefs.dictationHotkey) : '' })} -
- )} - {!loadError && filtered.map(s => ( - - ))} -
- - )} - - {(!mobile || mobileDetailOpen) && ( - - {item ? ( - <> - {mobile && ( -
- setMobileDetailOpen(false)}> - {t('history.backToList')} - -
- )} -
-
+// History.tsx — 接 Tauri 后端 list_history / delete_history_entry / clear_history。 +// 真实数据来自 ~/Library/Application Support/OpenLess/history.json。 + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Icon } from '../components/Icon'; +import { Tooltip } from '../components/Tooltip'; +import { detectOS } from '../components/WindowChrome'; +import { formatComboLabel } from '../lib/hotkey'; +import { clearHistory, deleteHistoryEntry, listHistory, readAudioRecording, retranscribeRecording, isTauri } from '../lib/ipc'; +import { useMobileLayout } from '../lib/useMobileLayout'; +import type { DictationSession, PolishMode } from '../lib/types'; +import { useHotkeySettings } from '../state/HotkeySettingsContext'; +import { Btn, Card, PageHeader, Pill } from './_atoms'; +import { chipSelectedStyle } from './settings/shared'; + +function useFilters(): Array<{ id: 'all' | PolishMode; label: string }> { + const { t } = useTranslation(); + return [ + { id: 'all', label: t('history.filterAll') }, + { id: 'raw', label: t('style.modes.raw.name') }, + { id: 'light', label: t('style.modes.light.name') }, + { id: 'structured', label: t('style.modes.structured.name') }, + { id: 'formal', label: t('style.modes.formal.name') }, + ]; +} + +function useModeLabel(): Record { + const { t } = useTranslation(); + return { + raw: t('style.modes.raw.name'), + light: t('style.modes.light.name'), + structured: t('style.modes.structured.name'), + formal: t('style.modes.formal.name'), + }; +} + +export function History() { + const { t } = useTranslation(); + const os = detectOS(); + const FILTERS = useFilters(); + const MODE_LABEL = useModeLabel(); + const [filter, setFilter] = useState<'all' | PolishMode>('all'); + const [query, setQuery] = useState(''); + const [debouncedQuery, setDebouncedQuery] = useState(''); + const [items, setItems] = useState([]); + const [selectedId, setSelectedId] = useState(null); + const [loading, setLoading] = useState(true); + const [loadError, setLoadError] = useState(null); + const [actionError, setActionError] = useState(null); + const [justCopied, setJustCopied] = useState(false); + const [justCopiedRaw, setJustCopiedRaw] = useState(false); + // 「重新转录」进行中：禁用按钮 + 显示「转录中…」，避免重复点击发起多次 ASR。 + const [retranscribing, setRetranscribing] = useState(false); + // 录音文件 lazily-detected missing 状态：retention / 条数 cap 清理后磁盘上 wav + // 可能已被删，但 history 条目 hasAudioRecording 仍写 true。任一组件 + // （播放 / 导出）首次 IPC 拿到 'recording not found' 时把 id 加进来， + // 之后渲染按钮的条件就转 false，避免反复点击得到同样的 error。 + // 修 pr_agent "Missing file check" 反馈。 + const [audioMissingIds, setAudioMissingIds] = useState>(() => new Set()); + const markAudioMissing = useCallback((id: string) => { + setAudioMissingIds(prev => { + if (prev.has(id)) return prev; + const next = new Set(prev); + next.add(id); + return next; + }); + }, []); + const { prefs } = useHotkeySettings(); + const mobile = useMobileLayout(); + const [mobileDetailOpen, setMobileDetailOpen] = useState(false); + + const refresh = useCallback(async () => { + setLoading(true); + setLoadError(null); + try { + const data = await listHistory(); + setItems(data); + setActionError(null); + setSelectedId(prev => (prev && data.some(s => s.id === prev) ? prev : data[0]?.id ?? null)); + } catch (error) { + console.error('[history] failed to load history', error); + setLoadError(errorMessage(error)); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void refresh(); + }, [refresh]); + + const searchInputRef = useRef(null); + const searchShortcut = os === 'mac' ? '⌘K' : 'Ctrl+K'; + + // 搜索词防抖：随输入实时更新 query，300ms 后落到 debouncedQuery 再过滤， + // 避免每个按键都重算整张列表（与 Marketplace 同模式）。 + useEffect(() => { + const id = window.setTimeout(() => setDebouncedQuery(query), 300); + return () => window.clearTimeout(id); + }, [query]); + + // ⌘K / Ctrl+K 聚焦搜索框（设计稿提示的快捷键）；⌘R / Ctrl+R 刷新历史列表 + // （与浏览器「重新加载」直觉一致）。preventDefault 拦掉 webview 默认的整页 + // reload，改为只重拉 listHistory，避免整个前端重挂载。 + useEffect(() => { + const onKeyDown = (e: KeyboardEvent) => { + if ((e.metaKey || e.ctrlKey) && (e.key === 'k' || e.key === 'K')) { + e.preventDefault(); + searchInputRef.current?.focus(); + return; + } + if ((e.metaKey || e.ctrlKey) && (e.key === 'r' || e.key === 'R')) { + e.preventDefault(); + void refresh(); + } + }; + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [refresh]); + + const filtered = useMemo(() => { + const byMode = filter === 'all' ? items : items.filter(s => s.mode === filter); + const q = debouncedQuery.trim().toLowerCase(); + if (!q) return byMode; + // 按原始转写 + 润色后文本匹配关键词，覆盖用户能想起的两种内容。 + return byMode.filter( + s => + s.rawTranscript.toLowerCase().includes(q) || + s.finalText.toLowerCase().includes(q), + ); + }, [items, filter, debouncedQuery]); + const item = useMemo( + () => filtered.find(s => s.id === selectedId) || filtered[0], + [filtered, selectedId], + ); + + const onClear = async () => { + if (items.length === 0) return; + if (!confirm(t('history.confirmClear', { count: items.length }))) return; + setActionError(null); + try { + await clearHistory(); + setItems([]); + setSelectedId(null); + } catch (error) { + console.error('[history] failed to clear history', error); + setActionError(t('history.clearFailed', { err: errorMessage(error) })); + } + }; + + const onDelete = async () => { + if (!item) return; + const deletedId = item.id; + setActionError(null); + try { + await deleteHistoryEntry(deletedId); + setItems(prev => prev.filter(s => s.id !== deletedId)); + setSelectedId(current => (current === deletedId ? null : current)); + } catch (error) { + console.error('[history] failed to delete history entry', error); + setActionError(t('history.deleteFailed', { err: errorMessage(error) })); + } + }; + + const onCopy = async () => { + if (!item) return; + try { + if (!navigator.clipboard?.writeText) { + throw new Error('clipboard unavailable'); + } + // 润色失败/未产出时 finalText 为空，回退到原文，避免「复制」按钮复制空字符串 + // 导致原文无法从 UI 取回（polish 失败时仍能拿到识别原文）。 + await navigator.clipboard.writeText(item.finalText.trim() ? item.finalText : item.rawTranscript); + setActionError(null); + setJustCopied(true); + window.setTimeout(() => setJustCopied(false), 1500); + } catch (error) { + console.error('[history] failed to copy entry', error); + setActionError(t('history.copyFailed', { err: errorMessage(error) })); + } + }; + + // 原文（识别结果）单独复制：润色失败或用户只想要未润色文本时使用。 + const onCopyRaw = async () => { + if (!item) return; + try { + if (!navigator.clipboard?.writeText) { + throw new Error('clipboard unavailable'); + } + await navigator.clipboard.writeText(item.rawTranscript); + setActionError(null); + setJustCopiedRaw(true); + window.setTimeout(() => setJustCopiedRaw(false), 1500); + } catch (error) { + console.error('[history] failed to copy raw transcript', error); + setActionError(t('history.copyFailed', { err: errorMessage(error) })); + } + }; + + const onExportAudio = async () => { + if (!item || !item.hasAudioRecording) return; + try { + // Wry/WebKit 中 data URL 的 可能不触发保存对话框，后端直接调系统对话框 + if (isTauri) { + const { invoke } = await import('@tauri-apps/api/core'); + await invoke('export_audio_recording', { sessionId: item.id }); + } else { + const dataUrl = await readAudioRecording(item.id); + if (!dataUrl || dataUrl === 'data:audio/wav;base64,') throw new Error('empty recording'); + const a = document.createElement('a'); + a.href = dataUrl; + a.download = `openless-recording-${item.id}.wav`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + } + setActionError(null); + } catch (error) { + console.error('[history] failed to export recording', error); + const msg = errorMessage(error); + if (isUserCancelled(msg)) { + setActionError(null); + return; + } + if (msg === 'recording export failed') { + setActionError(t('history.exportError')); + return; + } + // wav 已被 retention / 条数 cap 清理：把按钮隐藏，不显示错误（用户没干错事）。 + if (msg.includes('recording not found') || msg.includes('not found')) { + markAudioMissing(item.id); + return; + } + setActionError(t('history.exportFailed', { err: msg })); + } + }; + + // 对一条「转录失败 / 没识别到语音」的历史用当前 ASR provider 重新转录（issue #613）。 + // 后端读 recordings/.wav → 重转 → 原地回写该条 rawTranscript/finalText、清 errorCode， + // 返回整条记录；前端据此局部刷新。失败保留 + 自动重试已让这些条目的录音留得住，这里给 + // 持久失败（重试也没救回来）一个手动重转入口。 + const onRetranscribe = async () => { + if (!item || !item.hasAudioRecording) return; + setRetranscribing(true); + setActionError(null); + try { + const updated = await retranscribeRecording(item.id); + setItems(prev => prev.map(s => (s.id === updated.id ? updated : s))); + } catch (error) { + console.error('[history] retranscribe failed', error); + const msg = errorMessage(error); + // wav 已被 retention / 条数 cap 清理：隐藏入口，不报错（用户没干错事）。 + if (msg.includes('recording not found') || msg.includes('not found')) { + markAudioMissing(item.id); + return; + } + setActionError(t('history.retranscribeFailed', { err: msg })); + } finally { + setRetranscribing(false); + } + }; + + return ( +
+ + void refresh()}>{t('common.refresh')} + {t('common.clear')} +
+ } + /> +
+ {( !mobile || !mobileDetailOpen) && ( + +
+
+ + setQuery(e.target.value)} + placeholder={t('history.searchPlaceholder', { shortcut: searchShortcut })} + aria-label={t('history.searchPlaceholder', { shortcut: searchShortcut })} + style={{ + flex: 1, minWidth: 0, + outline: 'none', border: 0, background: 'transparent', + fontSize: 12, color: 'var(--ol-ink-1)', fontFamily: 'inherit', + }} + /> +
+
+ {t('history.summary', { total: items.length, shown: filtered.length })} +
+
+ {FILTERS.map(f => ( + + ))} +
+
+
+ {actionError && ( +
+ {actionError} +
+ )} + {loading &&
{t('common.loading')}
} + {!loading && loadError && ( +
+ {t('history.loadFailed', { err: loadError })} + void refresh()}>{t('history.retry')} +
+ )} + {!loading && !loadError && filtered.length === 0 && ( +
+ {debouncedQuery.trim() + ? t('history.searchNoMatch', { query: debouncedQuery.trim() }) + : t('history.empty', { trigger: prefs ? formatComboLabel(prefs.dictationHotkey) : '' })} +
+ )} + {!loadError && filtered.map(s => ( + + ))} +
+ + )} + + {(!mobile || mobileDetailOpen) && ( + + {item ? ( + <> + {mobile && ( +
+ setMobileDetailOpen(false)}> + {t('history.backToList')} + +
+ )} +
+
{formatTime(item.createdAt)} {MODE_LABEL[item.mode]} - {/* 「录音」前缀：与下方识别/润色耗时区分——录音时长发生在松键前， - 不该与流水线各步耗时加总（用户反馈"时间对不上"）。 */} - {t('history.recorded', { duration: formatDuration(item.durationMs, t) })} -
-
- {item.hasAudioRecording && !audioMissingIds.has(item.id) && ( - void onExportAudio()}>{t('history.exportRecording')} + {item.pipelineMode === 'multimodal' && ( + {t('history.multimodalPipeline')} )} + {/* 「录音」前缀：与下方识别/润色耗时区分——录音时长发生在松键前， + 不该与流水线各步耗时加总（用户反馈"时间对不上"）。 */} + {t('history.recorded', { duration: formatDuration(item.durationMs, t) })} +
+
+ {item.hasAudioRecording && !audioMissingIds.has(item.id) && ( + void onExportAudio()}>{t('history.exportRecording')} + )} {item.hasAudioRecording && !audioMissingIds.has(item.id) + && item.pipelineMode !== 'multimodal' && (item.errorCode === 'transcribeFailed' || item.errorCode === 'emptyTranscript') && ( - void onRetranscribe()}> - {retranscribing ? t('history.retranscribing') : t('history.retranscribe')} - - )} - {t('common.delete')} -
-
- {item.hasAudioRecording && !audioMissingIds.has(item.id) && ( - markAudioMissing(item.id)} - key={item.id} - /> - )} -
-
-
- {t('history.rawLabel')} - {item.rawTranscript && ( - void onCopyRaw()}> - {justCopiedRaw ? t('common.copied') : t('common.copy')} - - )} -
-

- {item.rawTranscript || t('history.rawEmpty')} -

-
-
-
- {MODE_LABEL[item.mode]} - void onCopy()}> - {justCopied ? t('common.copied') : t('common.copy')} - -
-

- {item.finalText} -

-
-
- {/* 流水线明细：识别 / 润色 / 插入 三步各占一行 —— 左列步骤名、中列 - provider·model（或插入目标），右列该步耗时/状态。旧历史没有模型与 - 耗时字段时对应行自动隐藏，只剩插入行 = 改版前的信息量。 */} -
- {(item.asrProvider || item.asrMs != null) && ( - <> - - - - {t('history.stepAsr')} - - - - - {[item.asrProvider, item.asrModel].filter(Boolean).join(' · ')} - - - {item.asrMs != null ? formatStepDuration(item.asrMs, t) : ''} - - - )} - {(item.llmProvider || item.llmModel || item.polishMs != null) && ( - <> - {t('history.stepPolish')} - - {[item.llmProvider, item.llmModel].filter(Boolean).join(' · ')} - - - {item.polishMs != null ? formatStepDuration(item.polishMs, t) : ''} - - - )} - {t('history.stepInsert')} - - {item.appName && <>{item.appName}{' · '}} - {t('history.chars', { count: item.finalText.length })} - {item.dictionaryEntryCount != null && item.dictionaryEntryCount > 0 && ( - <>{' · '}{t('history.vocabHits', { count: item.dictionaryEntryCount })} - )} - - { - item.insertStatus === 'inserted' - ? t('history.inserted') - : item.insertStatus === 'pasteSent' - ? t('history.pasteSent') - : item.insertStatus === 'copiedFallback' - ? t('history.copiedFallback', { shortcut: os === 'mac' ? '⌘V' : 'Ctrl+V' }) - : t('history.insertFailed') - } -
- - ) : ( -
- {loading ? t('common.loading') : loadError ? t('history.loadFailed', { err: loadError }) : t('history.selectHint')} -
- )} - - )} -
-
- ); -} - -function errorMessage(error: unknown): string { - if (typeof error === 'string') return error; - if (error instanceof Error) return error.message; - return String(error); -} - -function isUserCancelled(message: string): boolean { - const normalized = message.trim().toLowerCase(); - return normalized === 'cancelled' - || normalized === 'canceled' - || normalized === 'user cancelled' - || normalized === 'user canceled'; -} - -/** 当 session.hasAudioRecording 为 true 时渲染：一个加载按钮 + 拿到字节后切换为 - * 原生 audio controls。Blob URL 在组件 unmount 时 revoke，避免泄漏。 - * `onMissing` 在后端返回 'recording not found'（wav 已被 prune）时触发，让父组件 - * 把按钮永久隐藏，避免用户继续点击得到同样错误。 */ -function AudioRecordingPlayer({ - sessionId, - onMissing, -}: { - sessionId: string; - onMissing?: () => void; -}) { - const { t } = useTranslation(); - const [blobUrl, setBlobUrl] = useState(null); - const [status, setStatus] = useState<'idle' | 'loading' | 'ready' | 'error'>('idle'); - const [errorText, setErrorText] = useState(null); - const mountedRef = useRef(true); - const blobUrlRef = useRef(null); - - // 组件 unmount 时释放 Blob URL，避免内存泄漏。 - useEffect(() => { - mountedRef.current = true; - return () => { - mountedRef.current = false; - if (blobUrlRef.current) { - URL.revokeObjectURL(blobUrlRef.current); - blobUrlRef.current = null; - } - }; - }, []); - - const clearBlobUrl = () => { - if (blobUrlRef.current) { - URL.revokeObjectURL(blobUrlRef.current); - blobUrlRef.current = null; - } - setBlobUrl(null); - }; - - const load = async () => { - setStatus('loading'); - setErrorText(null); - try { - const dataUrl = await readAudioRecording(sessionId); - if (!mountedRef.current) return; - if (!dataUrl || dataUrl === 'data:audio/wav;base64,') throw new Error('empty recording'); - // WebKitGTK
+
+ {item.hasAudioRecording && !audioMissingIds.has(item.id) && ( + markAudioMissing(item.id)} + key={item.id} + /> + )} +
+
+
+ {t('history.rawLabel')} + {item.rawTranscript && ( + void onCopyRaw()}> + {justCopiedRaw ? t('common.copied') : t('common.copy')} + + )} +
+

+ {item.rawTranscript || t('history.rawEmpty')} +

+
+
+
+ {MODE_LABEL[item.mode]} + void onCopy()}> + {justCopied ? t('common.copied') : t('common.copy')} + +
+

+ {item.finalText} +

+
+
+ {/* 流水线明细：识别 / 润色 / 插入 三步各占一行 —— 左列步骤名、中列 + provider·model（或插入目标），右列该步耗时/状态。旧历史没有模型与 + 耗时字段时对应行自动隐藏，只剩插入行 = 改版前的信息量。 */} +
+ {(item.asrProvider || item.asrMs != null) && ( + <> + + + + {t('history.stepAsr')} + + + + + {[item.asrProvider, item.asrModel].filter(Boolean).join(' · ')} + + + {item.asrMs != null ? formatStepDuration(item.asrMs, t) : ''} + + + )} + {(item.llmProvider || item.llmModel || item.polishMs != null) && ( + <> + {t('history.stepPolish')} + + {[item.llmProvider, item.llmModel].filter(Boolean).join(' · ')} + + + {item.polishMs != null ? formatStepDuration(item.polishMs, t) : ''} + + + )} + {t('history.stepInsert')} + + {item.appName && <>{item.appName}{' · '}} + {t('history.chars', { count: item.finalText.length })} + {item.dictionaryEntryCount != null && item.dictionaryEntryCount > 0 && ( + <>{' · '}{t('history.vocabHits', { count: item.dictionaryEntryCount })} + )} + + { + item.insertStatus === 'inserted' + ? t('history.inserted') + : item.insertStatus === 'pasteSent' + ? t('history.pasteSent') + : item.insertStatus === 'copiedFallback' + ? t('history.copiedFallback', { shortcut: os === 'mac' ? '⌘V' : 'Ctrl+V' }) + : t('history.insertFailed') + } +
+ + ) : ( +
+ {loading ? t('common.loading') : loadError ? t('history.loadFailed', { err: loadError }) : t('history.selectHint')} +
+ )} + + )} + + + ); +} + +function errorMessage(error: unknown): string { + if (typeof error === 'string') return error; + if (error instanceof Error) return error.message; + return String(error); +} + +function isUserCancelled(message: string): boolean { + const normalized = message.trim().toLowerCase(); + return normalized === 'cancelled' + || normalized === 'canceled' + || normalized === 'user cancelled' + || normalized === 'user canceled'; +} + +/** 当 session.hasAudioRecording 为 true 时渲染：一个加载按钮 + 拿到字节后切换为 + * 原生 audio controls。Blob URL 在组件 unmount 时 revoke，避免泄漏。 + * `onMissing` 在后端返回 'recording not found'（wav 已被 prune）时触发，让父组件 + * 把按钮永久隐藏，避免用户继续点击得到同样错误。 */ +function AudioRecordingPlayer({ + sessionId, + onMissing, +}: { + sessionId: string; + onMissing?: () => void; +}) { + const { t } = useTranslation(); + const [blobUrl, setBlobUrl] = useState(null); + const [status, setStatus] = useState<'idle' | 'loading' | 'ready' | 'error'>('idle'); + const [errorText, setErrorText] = useState(null); + const mountedRef = useRef(true); + const blobUrlRef = useRef(null); + + // 组件 unmount 时释放 Blob URL，避免内存泄漏。 + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + if (blobUrlRef.current) { + URL.revokeObjectURL(blobUrlRef.current); + blobUrlRef.current = null; + } + }; + }, []); + + const clearBlobUrl = () => { + if (blobUrlRef.current) { + URL.revokeObjectURL(blobUrlRef.current); + blobUrlRef.current = null; + } + setBlobUrl(null); + }; + + const load = async () => { + setStatus('loading'); + setErrorText(null); + try { + const dataUrl = await readAudioRecording(sessionId); + if (!mountedRef.current) return; + if (!dataUrl || dataUrl === 'data:audio/wav;base64,') throw new Error('empty recording'); + // WebKitGTK