Skip to content

feat(pipeline): 多模态识别管线（实验性），传统/多模态模式与 omni 凭据隔离 (#902) #924

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
H-Chris233 wants to merge 1 commit into
base: beta
Choose a base branch
from
+3,655 −1,531
Open

feat(pipeline): 多模态识别管线（实验性），传统/多模态模式与 omni 凭据隔离 (#902) #924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 49 additions & 3 deletions openless-all/app/src-tauri/src/commands/credentials.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,29 @@ use super::*;

const LLM_EXTRA_HEADERS_ACCOUNT: &str = "ark.extra_headers";
const LLM_TEMPERATURE_ACCOUNT: &str = "ark.temperature";
const OMNI_EXTRA_HEADERS_ACCOUNT: &str = "omni.extra_headers";
const OMNI_TEMPERATURE_ACCOUNT: &str = "omni.temperature";

#[tauri::command]
pub async fn get_credentials() -> Result<CredentialsStatus, String> {
tauri::async_runtime::spawn_blocking(|| {
let snap = CredentialsVault::snapshot();
let active_asr_provider = CredentialsVault::get_active_asr();
let active_llm_provider = CredentialsVault::get_active_llm();
let pipeline_mode = PreferencesStore::new()
.map(|store| store.get().pipeline_mode)
.unwrap_or(crate::types::PipelineMode::Traditional);
let volcengine_configured = volcengine_configured(&snap);
let asr_configured = asr_configured_for_provider(&active_asr_provider, &snap);
let llm_configured = llm_configured_for_provider(&active_llm_provider, &snap);
let omni_configured = omni_configured_for_active_provider(&snap);
CredentialsStatus {
active_asr_provider,
active_llm_provider,
pipeline_mode,
asr_configured,
llm_configured,
omni_configured,
volcengine_configured,
ark_configured: llm_configured,
}
Expand Down Expand Up @@ -136,6 +144,18 @@ fn configured(field: &Option<String>) -> bool {
.unwrap_or(false)
}

/// 多模态（Omni）模型是否已配置：OpenAI 兼容通道要求 API Key + Base URL + Model；
/// Gemini 通道要求 API Key + Model（Base URL 为空时后端走官方默认）。
pub(crate) fn omni_configured_for_active_provider(snap: &CredentialsSnapshot) -> bool {
let provider = &snap.active_omni_provider;
let has_api_key = configured(&snap.omni_api_key);
let has_model = configured(&snap.omni_model);
if provider == "gemini" {
return has_api_key && has_model;
}
has_api_key && configured(&snap.omni_endpoint) && has_model
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg(not(mobile))]
pub(crate) struct LocalAsrReleasePlan {
Expand Down Expand Up @@ -189,7 +209,9 @@ pub async fn set_credential(
ensure_main_window(&window)?;
let extra_headers = account == LLM_EXTRA_HEADERS_ACCOUNT;
let temperature = account == LLM_TEMPERATURE_ACCOUNT;
let parsed = if extra_headers || temperature {
let omni_extra_headers = account == OMNI_EXTRA_HEADERS_ACCOUNT;
let omni_temperature = account == OMNI_TEMPERATURE_ACCOUNT;
let parsed = if extra_headers || temperature || omni_extra_headers || omni_temperature {
None
} else {
Some(parse_account(&account)?)
Expand All @@ -200,7 +222,14 @@ pub async fn set_credential(
.map_err(|e| e.to_string());
}
if temperature {
return CredentialsVault::set_active_llm_temperature(&value)
return CredentialsVault::set_active_llm_temperature(&value).map_err(|e| e.to_string());
}
if omni_extra_headers {
return CredentialsVault::set_active_omni_extra_headers_json(&value)
.map_err(|e| e.to_string());
}
if omni_temperature {
return CredentialsVault::set_active_omni_temperature(&value)
.map_err(|e| e.to_string());
}
let acc = parsed.expect("non-extra credential account must be parsed");
Expand Down Expand Up @@ -304,6 +333,11 @@ pub fn set_active_llm_provider(provider: String) -> Result<(), String> {
CredentialsVault::set_active_llm_provider(&provider).map_err(|e| e.to_string())
}

#[tauri::command]
pub fn set_active_omni_provider(provider: String) -> Result<(), String> {
CredentialsVault::set_active_omni_provider(&provider).map_err(|e| e.to_string())
}

/// 读出某个账号的实际值（用于设置页预填表单）。
/// 凭据来自系统凭据库；只允许主设置窗口读取 raw secret，避免胶囊 / QA 等辅助窗口默认暴露。
#[tauri::command]
Expand All @@ -315,7 +349,9 @@ pub async fn read_credential(
ensure_main_window(&window)?;
let extra_headers = account == LLM_EXTRA_HEADERS_ACCOUNT;
let temperature = account == LLM_TEMPERATURE_ACCOUNT;
let parsed = if extra_headers || temperature {
let omni_extra_headers = account == OMNI_EXTRA_HEADERS_ACCOUNT;
let omni_temperature = account == OMNI_TEMPERATURE_ACCOUNT;
let parsed = if extra_headers || temperature || omni_extra_headers || omni_temperature {
None
} else {
Some(parse_account(&account)?)
Expand All @@ -328,6 +364,13 @@ pub async fn read_credential(
if temperature {
return Ok(CredentialsVault::get_active_llm_temperature_string());
}
if omni_extra_headers {
return CredentialsVault::get_active_omni_extra_headers_json()
.map_err(|e| e.to_string());
}
if omni_temperature {
return Ok(CredentialsVault::get_active_omni_temperature_string());
}
let acc = parsed.expect("non-extra credential account must be parsed");
if let Some(provider) = provider {
CredentialsVault::get_for_asr_provider(&provider, acc).map_err(|e| e.to_string())
Expand Down Expand Up @@ -364,6 +407,9 @@ fn parse_account(s: &str) -> Result<CredentialAccount, String> {
"asr.advanced_config" => Ok(CredentialAccount::AsrAdvancedConfig),
"xfyun.app_id" => Ok(CredentialAccount::XfyunAppId),
"xfyun.api_key" => Ok(CredentialAccount::XfyunApiKey),
"omni.api_key" => Ok(CredentialAccount::OmniApiKey),
"omni.endpoint" => Ok(CredentialAccount::OmniEndpoint),
"omni.model" => Ok(CredentialAccount::OmniModel),
_ => Err(format!("unknown account: {s}")),
}
}
29 changes: 20 additions & 9 deletions openless-all/app/src-tauri/src/commands/history.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,17 @@ pub async fn read_audio_recording(session_id: String) -> Result<String, String>
format!("read wav failed: {e}")
}
})?;
log::info!("[history] read_audio_recording id={session_id} bytes={} head={:?}", data.len(), &data.get(..16));
log::info!(
"[history] read_audio_recording id={session_id} bytes={} head={:?}",
data.len(),
&data.get(..16)
);
let b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &data);
let data_url = format!("data:audio/wav;base64,{b64}");
log::info!("[history] read_audio_recording data_url_len={}", data_url.len());
log::info!(
"[history] read_audio_recording data_url_len={}",
data_url.len()
);
Ok(data_url)
}

Expand Down Expand Up @@ -122,9 +129,7 @@ fn export_recording_to_destination(
}
}

let destination = file_path
.into_path()
.map_err(export_recording_failed)?;
let destination = file_path.into_path().map_err(export_recording_failed)?;
copy_recording_to_path(source, &destination)?;
Ok(destination.to_string_lossy().into_owned())
}
Expand All @@ -151,7 +156,8 @@ fn copy_recording_to_path(
destination: &std::path::Path,
) -> Result<(), String> {
let mut source_file = open_recording_source(source)?;
let mut destination_file = std::fs::File::create(destination).map_err(export_recording_failed)?;
let mut destination_file =
std::fs::File::create(destination).map_err(export_recording_failed)?;
std::io::copy(&mut source_file, &mut destination_file)
.map(|_| ())
.map_err(export_recording_failed)
Expand All @@ -172,7 +178,9 @@ fn copy_recording_to_mobile_url(
Ok(file) => file,
Err(error) => {
#[cfg(target_os = "ios")]
let _ = app.fs().stop_accessing_security_scoped_resource(destination.clone());
let _ = app
.fs()
.stop_accessing_security_scoped_resource(destination.clone());
return Err(export_recording_failed(error));
}
};
Expand Down Expand Up @@ -256,7 +264,6 @@ pub async fn retranscribe_recording(
Ok(entry)
}


/// 把一次重转录的结果落到既有历史条目上（纯函数，供单测覆盖契约）：
/// - 只更新转写结果并清除失败标记。insert_status 保持原值——重新转录不向光标落字，
/// 没有可表达「已转写未落字」的状态，清掉 error_code 即足以标记不再是失败条目。
Expand Down Expand Up @@ -307,6 +314,7 @@ mod retranscribe_tests {
asr_model: Some("volc.seedasr.sauc.duration".into()),
llm_provider: Some("ark".into()),
llm_model: Some("deepseek-v3-2".into()),
pipeline_mode: None,
asr_ms: Some(15000),
polish_ms: Some(1200),
}
Expand All @@ -325,7 +333,10 @@ mod retranscribe_tests {
assert_eq!(entry.final_text, "重转出来的文本");
assert_eq!(entry.error_code, None, "重转成功应清除失败标记");
// ASR 归因换成本次重转的构建时快照。
assert_eq!(entry.asr_provider.as_deref(), Some("bailian-qwen3-realtime"));
assert_eq!(
entry.asr_provider.as_deref(),
Some("bailian-qwen3-realtime")
);
assert_eq!(entry.asr_model.as_deref(), Some("qwen3-asr-flash-realtime"));
assert_eq!(entry.asr_ms, Some(480));
// 重转没有润色环节：旧 LLM 元数据不得残留在新转写结果上。
Expand Down
47 changes: 41 additions & 6 deletions openless-all/app/src-tauri/src/commands/providers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ pub async fn validate_provider_credentials(kind: String) -> Result<ProviderCheck
"asr" => validate_asr_provider()
.await
.map(|()| ProviderCheckResult { ok: true }),
"omni" => validate_omni_provider()
.await
.map(|()| ProviderCheckResult { ok: true }),
_ => Err(format!("unknown provider kind: {kind}")),
}
}
Expand Down Expand Up @@ -129,6 +132,12 @@ fn read_openai_provider_config(kind: &str) -> Result<ProviderConfig, String> {
CredentialsVault::get_active_asr()
!= crate::coordinator::OPENAI_COMPATIBLE_ASR_PROVIDER_ID,
),
// 多模态（Omni）模型：独立命名空间，OpenAI 兼容通道要求 API Key + Base URL。
"omni" => (
CredentialAccount::OmniApiKey,
CredentialAccount::OmniEndpoint,
true,
),
_ => return Err(format!("unknown provider kind: {kind}")),
};
let api_key = CredentialsVault::get(api_key_account)
Expand All @@ -146,6 +155,15 @@ fn read_openai_provider_config(kind: &str) -> Result<ProviderConfig, String> {
CredentialsVault::get_active_llm_temperature(),
),
)
} else if kind == "omni" {
let active_omni = CredentialsVault::get_active_omni();
(
CredentialsVault::get_active_omni_extra_headers(),
openai_compatible_temperature_for_provider(
&active_omni,
CredentialsVault::get_active_omni_temperature(),
),
)
} else {
(HashMap::new(), None)
};
Expand Down Expand Up @@ -246,6 +264,18 @@ fn provider_llm_error_message(error: LLMError) -> String {
}
}

/// 多模态（Omni）模型连通性验证：真发一次纯文本请求（无音频），走与运行期
/// 完全相同的 provider 构建与请求路径，避免「验证通过但真实调用失败」。
async fn validate_omni_provider() -> Result<(), String> {
let provider =
crate::coordinator::build_active_omni_provider(false).map_err(|e| e.to_string())?;
provider
.complete("验证连接", "ping", None)
.await
.map(|_| ())
.map_err(provider_llm_error_message)
}

async fn validate_asr_provider() -> Result<(), String> {
let active_asr = CredentialsVault::get_active_asr();
if active_asr_is_keyless_for_validation(&active_asr) {
Expand Down Expand Up @@ -746,8 +776,7 @@ async fn validate_asr_transcription(
request.json(&body)
}
};
match request.send().await
{
match request.send().await {
Ok(resp) => break resp,
Err(e) if e.is_timeout() => return Err("providerRequestTimeout".to_string()),
Err(e) if (e.is_connect() || e.is_request()) && attempt < MAX_ATTEMPTS => {
Expand Down Expand Up @@ -1241,9 +1270,12 @@ mod tests {
stream.write_all(response.as_bytes()).await.unwrap();
});
let target_server = tokio::spawn(async move {
tokio::time::timeout(std::time::Duration::from_millis(500), target_listener.accept())
.await
.is_ok()
tokio::time::timeout(
std::time::Duration::from_millis(500),
target_listener.accept(),
)
.await
.is_ok()
});

let error = send_dashscope_multimodal_validation(
Expand All @@ -1256,7 +1288,10 @@ mod tests {

redirect_server.await.unwrap();
assert_eq!(error, "providerHttpStatus:302");
assert!(!target_server.await.unwrap(), "validation followed redirect");
assert!(
!target_server.await.unwrap(),
"validation followed redirect"
);
}

#[test]
Expand Down
Loading