From b8c12d9eb636d23887a69281202d2b67f29270b9 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 25 Aug 2025 18:04:20 +0800 Subject: [PATCH 01/18] texture builder abstraction --- apps/desktop/src-tauri/src/camera.rs | 221 +++++++++++++++------------ 1 file changed, 127 insertions(+), 94 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 3013444078..40fc8495be 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -18,11 +18,11 @@ use std::{ thread, time::Duration, }; -use tauri::{LogicalPosition, LogicalSize, Manager, PhysicalSize, WebviewWindow, Wry}; +use tauri::{LogicalPosition, LogicalSize, Manager, PhysicalSize, WebviewWindow, Window, Wry}; use tauri_plugin_store::Store; use tokio::sync::{broadcast, oneshot}; use tracing::error; -use wgpu::{CompositeAlphaMode, SurfaceTexture}; +use wgpu::{CompositeAlphaMode, Surface, SurfaceTexture}; static TOOLBAR_HEIGHT: f32 = 56.0; // also defined in Typescript @@ -202,8 +202,8 @@ impl CameraPreview { .get_current_texture() .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) { - let output_width = 1280; - let output_height = (1280.0 / camera_aspect_ratio) as u32; + let output_width = 100; // 1280; + let output_height = 100; // (1280.0 / camera_aspect_ratio) as u32; let new_texture_value = if let Some(frame) = frame { if loading { @@ -245,12 +245,27 @@ impl CameraPreview { None // This will reuse the existing texture }; - renderer.render( - surface, - new_texture_value.as_ref().map(|(b, s)| (&**b, *s)), - output_width, - output_height, - ); + // TODO: Remove this option but does that cause issues??? + if let Some((buffer, stride)) = + new_texture_value.as_ref().map(|(b, s)| (&**b, *s)) + { + renderer + .texture + .get_or_init((output_width, output_height), || { + PreparedTexture::init( + renderer.device.clone(), + renderer.queue.clone(), + &renderer.sampler, + &renderer.bind_group_layout, + renderer.uniform_bind_group.clone(), + renderer.render_pipeline.clone(), + output_width, + output_height, + ) + }) + .render(&surface, buffer, stride); + surface.present(); + } } if !window_visible { @@ -307,7 +322,7 @@ struct Renderer { state: CameraWindowState, frame_info: Cached<(format::Pixel, u32, u32)>, surface_size: Cached<(u32, u32)>, - texture: Cached<(u32, u32), (wgpu::Texture, wgpu::TextureView, wgpu::BindGroup)>, + texture: Cached<(u32, u32), PreparedTexture>, } impl Renderer { @@ -684,22 +699,93 @@ impl Renderer { bytemuck::cast_slice(&[camera_uniforms]), ); } +} - /// Render the camera preview to the window. - fn render( - &mut self, - surface: SurfaceTexture, - new_texture_value: Option<(&[u8], u32)>, +fn render_solid_frame(color: [u8; 4], width: u32, height: u32) -> (Vec, u32) { + let pixel_count = (height * width) as usize; + let buffer: Vec = color + .iter() + .cycle() + .take(pixel_count * 4) + .copied() + .collect(); + + (buffer, 4 * width) +} + +pub struct PreparedTexture { + texture: wgpu::Texture, + texture_view: wgpu::TextureView, + bind_group: wgpu::BindGroup, + uniform_bind_group: wgpu::BindGroup, + render_pipeline: wgpu::RenderPipeline, + device: wgpu::Device, + queue: wgpu::Queue, + width: u32, + height: u32, +} + +impl PreparedTexture { + pub fn init( + device: wgpu::Device, + queue: wgpu::Queue, + sampler: &wgpu::Sampler, + bind_group_layout: &wgpu::BindGroupLayout, + uniform_bind_group: wgpu::BindGroup, + render_pipeline: wgpu::RenderPipeline, width: u32, height: u32, - ) { + ) -> Self { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + + let texture_view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Texture Bind Group"), + layout: bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&texture_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }); + + Self { + texture, + texture_view, + bind_group, + uniform_bind_group, + render_pipeline, + device, + queue, + width, + height, + } + } + + pub fn render(&self, surface: &SurfaceTexture, buffer: &[u8], stride: u32) { let surface_view = surface .texture .create_view(&wgpu::TextureViewDescriptor::default()); - // let surface_width = surface.texture.width(); - // let surface_height = surface.texture.height(); - let mut encoder = self .device .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None }); @@ -709,8 +795,7 @@ impl Renderer { label: None, color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &surface_view, - // depth_slice: None, - resolve_target: None, // Some(&surface_view), + resolve_target: None, ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color { r: 0.0, @@ -726,88 +811,36 @@ impl Renderer { occlusion_query_set: None, }); - // Get or reinitialize the texture if necessary - let (texture, _, bind_group) = &*self.texture.get_or_init((width, height), || { - let texture = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("Camera Texture"), - size: wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rgba8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - - let texture_view = texture.create_view(&wgpu::TextureViewDescriptor::default()); - - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("Texture Bind Group"), - layout: &self.bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&texture_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(&self.sampler), - }, - ], - }); - - (texture, texture_view, bind_group) - }); - - if let Some((buffer, stride)) = new_texture_value { - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - buffer, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(stride), - rows_per_image: Some(height), - }, - wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - ); - } + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &self.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + buffer, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(stride), + rows_per_image: Some(self.height), + }, + wgpu::Extent3d { + width: self.width, + height: self.height, + depth_or_array_layers: 1, + }, + ); render_pass.set_pipeline(&self.render_pipeline); - render_pass.set_bind_group(0, bind_group, &[]); + render_pass.set_bind_group(0, &self.bind_group, &[]); render_pass.set_bind_group(1, &self.uniform_bind_group, &[]); render_pass.draw(0..6, 0..1); } self.queue.submit(Some(encoder.finish())); - surface.present(); } } -fn render_solid_frame(color: [u8; 4], width: u32, height: u32) -> (Vec, u32) { - let pixel_count = (height * width) as usize; - let buffer: Vec = color - .iter() - .cycle() - .take(pixel_count * 4) - .copied() - .collect(); - - (buffer, 4 * width) -} - struct Cached { value: Option<(K, V)>, } From 6674e9aca1095f81c438f43dc4ba6c0ae22505ec Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 25 Aug 2025 18:06:32 +0800 Subject: [PATCH 02/18] rip out loading state for now --- apps/desktop/src-tauri/src/camera.rs | 18 ------------------ apps/desktop/src-tauri/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 40fc8495be..5a7ede7c2d 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -61,7 +61,6 @@ pub struct CameraPreview { broadcast::Sender>, broadcast::Receiver>, ), - loading: Arc, store: Arc>, } @@ -69,7 +68,6 @@ impl CameraPreview { pub fn init(manager: &impl Manager) -> tauri_plugin_store::Result { Ok(Self { reconfigure: broadcast::channel(1), - loading: Arc::new(AtomicBool::new(false)), store: tauri_plugin_store::StoreBuilder::new(manager, "cameraPreview").build()?, }) } @@ -79,13 +77,10 @@ impl CameraPreview { window: WebviewWindow, camera_rx: Receiver, ) -> anyhow::Result<()> { - self.loading.store(true, Ordering::Relaxed); - let mut renderer = Renderer::init(window.clone()).await?; let store = self.store.clone(); let mut reconfigure = self.reconfigure.1.resubscribe(); - let loading_state = self.loading.clone(); thread::spawn(move || { let mut window_visible = false; let mut first = true; @@ -206,11 +201,6 @@ impl CameraPreview { let output_height = 100; // (1280.0 / camera_aspect_ratio) as u32; let new_texture_value = if let Some(frame) = frame { - if loading { - loading_state.store(false, Ordering::Relaxed); - loading = false; - } - let resampler_frame = resampler_frame .get_or_init((output_width, output_height), frame::Video::empty); @@ -290,14 +280,6 @@ impl CameraPreview { Ok(()) } - /// Wait for the camera to load. - pub async fn wait_for_camera_to_load(&self) { - // The webview is generally slow to load so it's rare this will actually loop. - while self.loading.load(Ordering::Relaxed) { - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - /// Update the size of the window. /// Using `window.outer_size` just never resolves when a native menu is open. pub fn update_window_size(&self, width: u32, height: u32) { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 2a68041111..d5e88dd03a 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1884,7 +1884,7 @@ async fn set_camera_preview_state( #[tauri::command] #[specta::specta] async fn await_camera_preview_ready(store: State<'_, CameraPreview>) -> Result { - store.wait_for_camera_to_load().await; + // store.wait_for_camera_to_load().await; // TODO: Reimplement this Ok(true) } From e6c20bc94030f173b958768b2b02ec2ad65df324 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 25 Aug 2025 18:12:06 +0800 Subject: [PATCH 03/18] init fallback texture on startup --- apps/desktop/src-tauri/src/camera.rs | 38 +++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 5a7ede7c2d..83554bdf12 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -84,7 +84,6 @@ impl CameraPreview { thread::spawn(move || { let mut window_visible = false; let mut first = true; - let mut loading = true; let mut window_size = None; let mut resampler_frame = Cached::default(); let mut aspect_ratio = None; @@ -101,6 +100,35 @@ impl CameraPreview { return; }; + if let Ok(surface) = renderer + .surface + .get_current_texture() + .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) + { + // TODO: Smaller??? + let output_width = 50; + let output_height = 50; + + let (buffer, stride) = render_solid_frame( + [0x11, 0x11, 0x11, 0xFF], // #111111 + output_width, + output_height, + ); + + let fallback_texture = PreparedTexture::init( + renderer.device.clone(), + renderer.queue.clone(), + &renderer.sampler, + &renderer.bind_group_layout, + renderer.uniform_bind_group.clone(), + renderer.render_pipeline.clone(), + output_width, + output_height, + ) + .render(&surface, &buffer, stride); + surface.present(); + } + while let Some((frame, reconfigure)) = block_on({ let camera_rx = &camera_rx; let reconfigure = &mut reconfigure; @@ -223,14 +251,6 @@ impl CameraPreview { resampler_frame.data(0).to_vec(), resampler_frame.stride(0) as u32, )) - } else if loading { - let (buffer, stride) = render_solid_frame( - [0x11, 0x11, 0x11, 0xFF], // #111111 - output_width, - output_height, - ); - - Some((buffer, stride)) } else { None // This will reuse the existing texture }; From 86083e443f451eb46a3b6a54b34c05812110f5a2 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 25 Aug 2025 23:15:09 +0800 Subject: [PATCH 04/18] overhaul --- apps/desktop/src-tauri/src/camera.rs | 776 +++++++++--------- .../desktop/src-tauri/src/deeplink_actions.rs | 7 +- apps/desktop/src-tauri/src/lib.rs | 98 +-- apps/desktop/src-tauri/src/recording.rs | 5 +- apps/desktop/src-tauri/src/windows.rs | 101 ++- apps/desktop/src/routes/camera.tsx | 2 - 6 files changed, 502 insertions(+), 487 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 83554bdf12..bd4f2524e9 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -1,28 +1,21 @@ -use anyhow::Context; +use anyhow::{Context, anyhow}; use cap_recording::feeds::RawCameraFrame; use ffmpeg::{ format::{self, Pixel}, frame, software::scaling, }; -use flume::Receiver; -use futures::{executor::block_on, future::Either}; use serde::{Deserialize, Serialize}; use specta::Type; -use std::{ - pin::pin, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - thread, - time::Duration, +use std::{sync::Arc, thread}; +use tauri::{LogicalPosition, LogicalSize, PhysicalSize, WebviewWindow}; +use tokio::{ + runtime::Runtime, + sync::{broadcast, oneshot}, + task::LocalSet, }; -use tauri::{LogicalPosition, LogicalSize, Manager, PhysicalSize, WebviewWindow, Window, Wry}; -use tauri_plugin_store::Store; -use tokio::sync::{broadcast, oneshot}; -use tracing::error; -use wgpu::{CompositeAlphaMode, Surface, SurfaceTexture}; +use tracing::{error, info}; +use wgpu::{CompositeAlphaMode, SurfaceTexture}; static TOOLBAR_HEIGHT: f32 = 56.0; // also defined in Typescript @@ -49,295 +42,150 @@ pub enum CameraPreviewShape { } #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize, Type)] -pub struct CameraWindowState { +pub struct CameraPreviewState { size: CameraPreviewSize, shape: CameraPreviewShape, mirrored: bool, } -pub struct CameraPreview { - #[allow(clippy::type_complexity)] - reconfigure: ( - broadcast::Sender>, - broadcast::Receiver>, +pub struct CameraPreviewManager { + store: Result>, String>, + preview: Option, + // TODO: Reusing flume channels can be unsafe as the frames will only + // go to a single receiver, not all of them. + channel: ( + flume::Sender, + flume::Receiver, ), - store: Arc>, } -impl CameraPreview { - pub fn init(manager: &impl Manager) -> tauri_plugin_store::Result { - Ok(Self { - reconfigure: broadcast::channel(1), - store: tauri_plugin_store::StoreBuilder::new(manager, "cameraPreview").build()?, - }) +impl CameraPreviewManager { + /// Create a new camera preview manager. + pub fn new(app: &tauri::AppHandle) -> Self { + Self { + store: tauri_plugin_store::StoreBuilder::new(app, "cameraPreview") + .build() + .map_err(|err| format!("Error initializing camera preview store: {err}")), + preview: None, + channel: flume::bounded(4), + } } - pub async fn init_preview_window( - &self, - window: WebviewWindow, - camera_rx: Receiver, - ) -> anyhow::Result<()> { - let mut renderer = Renderer::init(window.clone()).await?; - - let store = self.store.clone(); - let mut reconfigure = self.reconfigure.1.resubscribe(); - thread::spawn(move || { - let mut window_visible = false; - let mut first = true; - let mut window_size = None; - let mut resampler_frame = Cached::default(); - let mut aspect_ratio = None; - let Ok(mut scaler) = scaling::Context::get( - Pixel::RGBA, - 1, - 1, - Pixel::RGBA, - 1, - 1, - scaling::Flags::empty(), - ) - .map_err(|err| error!("Error initializing ffmpeg scaler: {err:?}")) else { - return; - }; - - if let Ok(surface) = renderer - .surface - .get_current_texture() - .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) - { - // TODO: Smaller??? - let output_width = 50; - let output_height = 50; - - let (buffer, stride) = render_solid_frame( - [0x11, 0x11, 0x11, 0xFF], // #111111 - output_width, - output_height, - ); - - let fallback_texture = PreparedTexture::init( - renderer.device.clone(), - renderer.queue.clone(), - &renderer.sampler, - &renderer.bind_group_layout, - renderer.uniform_bind_group.clone(), - renderer.render_pipeline.clone(), - output_width, - output_height, - ) - .render(&surface, &buffer, stride); - surface.present(); - } - - while let Some((frame, reconfigure)) = block_on({ - let camera_rx = &camera_rx; - let reconfigure = &mut reconfigure; - - async { - // Triggers the first paint - if first { - // We don't set `first = false` as that is done within the loop. - return Some((None, true)); - } - - match futures::future::select( - pin!(camera_rx.recv_async()), - pin!(reconfigure.recv()), - ) - .await - { - Either::Left((frame, _)) => frame.ok().map(|f| (Some(f.frame), false)), - Either::Right((event, _)) => { - if let Ok(Some((width, height))) = event { - window_size = Some((width, height)); - } - - Some((None, true)) - } - } - } - }) { - let window_resize_required = - if reconfigure && renderer.refresh_state(&store) || first { - first = false; - renderer.update_state_uniforms(); - true - } else if let Some(frame) = frame.as_ref() - && renderer.frame_info.update_key_and_should_init(( - frame.format(), - frame.width(), - frame.height(), - )) - { - aspect_ratio = Some(frame.width() as f32 / frame.height() as f32); - - true - } else { - false - }; - - let camera_aspect_ratio = - aspect_ratio.unwrap_or(if renderer.state.shape == CameraPreviewShape::Full { - 16.0 / 9.0 - } else { - 1.0 - }); - - if window_resize_required { - renderer.update_camera_aspect_ratio_uniforms(camera_aspect_ratio); - - match renderer.resize_window(camera_aspect_ratio) { - Ok(size) => window_size = Some(size), - Err(err) => { - error!("Error updating window size: {err:?}"); - continue; - } - } - } - - let (window_width, window_height) = match window_size { - Some(s) => s, - // Calling `window.outer_size` will hang when a native menu is opened. - // So we only callback to it if absolute required as it could randomly hang. - None => match renderer - .window - .inner_size() - .and_then(|size| Ok(size.to_logical(renderer.window.scale_factor()?))) - { - Ok(size) => { - window_size = Some((size.width, size.height)); - (size.width, size.height) - } - Err(err) => { - error!("Error getting window size: {err:?}"); - continue; - } - }, - }; + /// Get the current state of the camera window. + pub fn get_state(&self) -> anyhow::Result { + Ok(self + .store + .as_ref() + .map_err(|err| anyhow!("{err}"))? + .get("state") + .and_then(|v| serde_json::from_value(v).ok().unwrap_or_default()) + .unwrap_or_default()) + } - if let Err(err) = renderer.reconfigure_gpu_surface(window_width, window_height) { - error!("Error reconfiguring GPU surface: {err:?}"); - continue; - } + /// Save the current state of the camera window. + pub fn set_state(&self, state: CameraPreviewState) -> anyhow::Result<()> { + let store = self.store.as_ref().map_err(|err| anyhow!("{err}"))?; + store.set("state", serde_json::to_value(&state)?); + store.save()?; + + if let Some(preview) = &self.preview { + preview + .reconfigure + .send(ReconfigureEvent::State(state)) + .map_err(|err| error!("Error asking camera preview to reconfigure: {err}")) + .ok(); + } - if let Ok(surface) = renderer - .surface - .get_current_texture() - .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) - { - let output_width = 100; // 1280; - let output_height = 100; // (1280.0 / camera_aspect_ratio) as u32; + Ok(()) + } - let new_texture_value = if let Some(frame) = frame { - let resampler_frame = resampler_frame - .get_or_init((output_width, output_height), frame::Video::empty); + pub fn attach(&self) -> flume::Sender { + // Drain the channel so when the preview is opened it doesn't show an old frame. + while let Ok(_) = self.channel.1.try_recv() {} - scaler.cached( - frame.format(), - frame.width(), - frame.height(), - format::Pixel::RGBA, - output_width, - output_height, - ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR, - ); + self.channel.0.clone() + } - if let Err(err) = scaler.run(&frame, resampler_frame) { - error!("Error rescaling frame with ffmpeg: {err:?}"); - continue; - } + pub fn is_initialized(&self) -> bool { + self.preview.is_some() + } - Some(( - resampler_frame.data(0).to_vec(), - resampler_frame.stride(0) as u32, - )) - } else { - None // This will reuse the existing texture - }; - - // TODO: Remove this option but does that cause issues??? - if let Some((buffer, stride)) = - new_texture_value.as_ref().map(|(b, s)| (&**b, *s)) - { - renderer - .texture - .get_or_init((output_width, output_height), || { - PreparedTexture::init( - renderer.device.clone(), - renderer.queue.clone(), - &renderer.sampler, - &renderer.bind_group_layout, - renderer.uniform_bind_group.clone(), - renderer.render_pipeline.clone(), - output_width, - output_height, - ) - }) - .render(&surface, buffer, stride); - surface.present(); - } - } + /// Initialize the camera preview for a specific Tauri window + pub async fn init_window(&mut self, window: WebviewWindow) -> anyhow::Result<()> { + let default_state = self + .get_state() + .map_err(|err| error!("Error getting camera preview state: {err}")) + .unwrap_or_default(); - if !window_visible { - window_visible = true; - if let Err(err) = renderer.window.show() { - error!("Failed to show camera preview window: {}", err); - } - } - } + let (reconfigure, reconfigure_rx) = broadcast::channel(1); + let mut renderer = + InitializedCameraPreview::init_wgpu(window.clone(), default_state).await?; + window.show().ok(); - window.close().ok(); + let camera_rx = self.channel.1.clone(); + let rt = Runtime::new().unwrap(); + thread::spawn(move || { + LocalSet::new().block_on(&rt, renderer.run(window, reconfigure_rx, camera_rx)) }); + self.preview = Some(InitializedCameraPreview { reconfigure }); + Ok(()) } - /// Save the current state of the camera window. - pub fn save(&self, state: &CameraWindowState) -> tauri_plugin_store::Result<()> { - self.store.set("state", serde_json::to_value(state)?); - self.store.save()?; - self.reconfigure.0.send(None).ok(); - Ok(()) + /// Called by Tauri's event loop in response to a window resize event. + /// In theory if we get the event loop right this isn't required, + /// but it means if we mistake we get a small glitch instead of it + /// being permanently incorrectly sized or scaled. + pub fn on_window_resize(&self, width: u32, height: u32) { + if let Some(preview) = &self.preview { + preview + .reconfigure + .send(ReconfigureEvent::WindowSize(width, height)) + .ok(); + } } - /// Update the size of the window. - /// Using `window.outer_size` just never resolves when a native menu is open. - pub fn update_window_size(&self, width: u32, height: u32) { - self.reconfigure.0.send(Some((width, height))).ok(); + /// Called by Tauri's event loop in response to a window destroy event. + pub fn on_window_close(&mut self) { + if let Some(preview) = self.preview.take() { + info!("Camera preview window closed."); + preview + .reconfigure + .send(ReconfigureEvent::Shutdown) + .map_err(|err| error!("Error sending camera preview shutdown event: {err}")) + .ok(); + } + + // Drain the channel so when the preview is opened it doesn't show it. + while let Ok(_) = self.channel.1.try_recv() {} } } -struct Renderer { - surface: wgpu::Surface<'static>, - surface_config: wgpu::SurfaceConfiguration, - render_pipeline: wgpu::RenderPipeline, - device: wgpu::Device, - queue: wgpu::Queue, - sampler: wgpu::Sampler, - bind_group_layout: wgpu::BindGroupLayout, - uniform_buffer: wgpu::Buffer, - window_uniform_buffer: wgpu::Buffer, - camera_uniform_buffer: wgpu::Buffer, - uniform_bind_group: wgpu::BindGroup, - window: tauri::WebviewWindow, +#[derive(Clone)] +enum ReconfigureEvent { + WindowSize(u32, u32), + State(CameraPreviewState), + Shutdown, +} - state: CameraWindowState, - frame_info: Cached<(format::Pixel, u32, u32)>, - surface_size: Cached<(u32, u32)>, - texture: Cached<(u32, u32), PreparedTexture>, +struct InitializedCameraPreview { + reconfigure: broadcast::Sender, } -impl Renderer { - /// Initialize a new renderer for a specific Tauri window. - async fn init(window: WebviewWindow) -> anyhow::Result { - let size = window - .inner_size() - .with_context(|| "Error getting the window size")? - .to_logical( - window - .scale_factor() - .with_context(|| "Error getting the window scale")?, - ); +impl InitializedCameraPreview { + async fn init_wgpu( + window: WebviewWindow, + default_state: CameraPreviewState, + ) -> anyhow::Result { + let aspect = if default_state.shape == CameraPreviewShape::Full { + 16.0 / 9.0 + } else { + 1.0 + }; + + let size = resize_window(&window, &default_state, aspect) + .context("Error resizing Tauri window")?; let (tx, rx) = oneshot::channel(); window @@ -443,8 +291,8 @@ impl Renderer { ], }); - let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("Uniform Buffer"), + let state_uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("State Uniform Buffer"), size: std::mem::size_of::() as u64, usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false, @@ -470,7 +318,7 @@ impl Renderer { entries: &[ wgpu::BindGroupEntry { binding: 0, - resource: uniform_buffer.as_entire_binding(), + resource: state_uniform_buffer.as_entire_binding(), }, wgpu::BindGroupEntry { binding: 1, @@ -534,8 +382,8 @@ impl Renderer { let surface_config = wgpu::SurfaceConfiguration { usage: wgpu::TextureUsages::RENDER_ATTACHMENT, format: swapchain_format, - width: size.width, - height: size.height, + width: size.0, + height: size.1, present_mode: wgpu::PresentMode::Fifo, alpha_mode, view_formats: vec![], @@ -554,7 +402,7 @@ impl Renderer { ..Default::default() }); - Ok(Self { + let mut renderer = Renderer { surface, surface_config, render_pipeline, @@ -562,147 +410,305 @@ impl Renderer { queue, sampler, bind_group_layout, - uniform_buffer, + state_uniform_buffer, window_uniform_buffer, camera_uniform_buffer, uniform_bind_group, - window, - - state: Default::default(), - frame_info: Cached::default(), - surface_size: Cached::default(), texture: Cached::default(), - }) - } + aspect_ratio: Cached::default(), + }; - /// Update the local cache of the camera state - fn refresh_state(&mut self, store: &Store) -> bool { - let current = self.state.clone(); + renderer.update_state_uniforms(&default_state); + renderer.sync_aspect_ratio_uniforms(aspect); + renderer.reconfigure_gpu_surface(size.0, size.1); - self.state = store - .get("state") - .and_then(|v| serde_json::from_value(v).ok()) - .unwrap_or_default(); + // We initialize and render a blank color fallback. + // This is shown until the camera initializes and the first frame is rendered. + if let Ok(surface) = renderer + .surface + .get_current_texture() + .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) + { + let output_width = 5; + let output_height = (5.0 * 1.7777778) as u32; // TODO + + let (buffer, stride) = render_solid_frame( + [0x11, 0x11, 0x11, 0xFF], // #111111 + output_width, + output_height, + ); + + PreparedTexture::init( + renderer.device.clone(), + renderer.queue.clone(), + &renderer.sampler, + &renderer.bind_group_layout, + renderer.uniform_bind_group.clone(), + renderer.render_pipeline.clone(), + output_width, + output_height, + ) + .render(&surface, &buffer, stride); + surface.present(); + + println!("1: {:?} {:?} {:?}", size.0, size.1, aspect); + } - current != self.state + Ok(renderer) } +} - /// Resize the OS window to the correct size - fn resize_window(&self, aspect: f32) -> tauri::Result<(u32, u32)> { - let base: f32 = if self.state.size == CameraPreviewSize::Sm { - 230.0 - } else { - 400.0 - }; - let window_width = if self.state.shape == CameraPreviewShape::Full { - if aspect >= 1.0 { base * aspect } else { base } - } else { - base - }; - let window_height = if self.state.shape == CameraPreviewShape::Full { - if aspect >= 1.0 { base } else { base / aspect } - } else { - base - } + TOOLBAR_HEIGHT; - - let (monitor_size, monitor_offset, monitor_scale_factor): ( - PhysicalSize, - LogicalPosition, - _, - ) = if let Some(monitor) = self.window.current_monitor()? { - let size = monitor.position().to_logical(monitor.scale_factor()); - (*monitor.size(), size, monitor.scale_factor()) - } else { - (PhysicalSize::new(640, 360), LogicalPosition::new(0, 0), 1.0) +struct Renderer { + surface: wgpu::Surface<'static>, + surface_config: wgpu::SurfaceConfiguration, + render_pipeline: wgpu::RenderPipeline, + device: wgpu::Device, + queue: wgpu::Queue, + sampler: wgpu::Sampler, + bind_group_layout: wgpu::BindGroupLayout, + state_uniform_buffer: wgpu::Buffer, + window_uniform_buffer: wgpu::Buffer, + camera_uniform_buffer: wgpu::Buffer, + uniform_bind_group: wgpu::BindGroup, + texture: Cached<(u32, u32), PreparedTexture>, + aspect_ratio: Cached, +} + +impl Renderer { + async fn run( + &mut self, + window: WebviewWindow, + mut reconfigure: broadcast::Receiver, + camera_rx: flume::Receiver, + ) { + // let mut window_size = (size.width, size.height); + let mut resampler_frame = Cached::default(); + let Ok(mut scaler) = scaling::Context::get( + Pixel::RGBA, + 1, + 1, + Pixel::RGBA, + 1, + 1, + scaling::Flags::empty(), + ) + .map_err(|err| error!("Error initializing ffmpeg scaler: {err:?}")) else { + return; }; - let x = (monitor_size.width as f64 / monitor_scale_factor - window_width as f64 - 100.0) - as u32 - + monitor_offset.x; - let y = (monitor_size.height as f64 / monitor_scale_factor - window_height as f64 - 100.0) - as u32 - + monitor_offset.y; + let mut seen_first_frame = false; + while let Some(event) = loop { + tokio::select! { + frame = camera_rx.recv_async() => break frame.ok().map(Ok), + result = reconfigure.recv() => { + if let Ok(result) = result { + break Some(Err(result)) + } else { + continue; + } + }, + } + } { + match event { + Ok(frame) => { + if !seen_first_frame { + seen_first_frame = true; + } + let aspect_ratio = frame.frame.width() as f32 / frame.frame.height() as f32; + self.sync_aspect_ratio_uniforms(aspect_ratio); + + println!("2: {:?}", aspect_ratio); - self.window - .set_size(LogicalSize::new(window_width, window_height))?; - self.window.set_position(LogicalPosition::new(x, y))?; + if let Ok(surface) = self.surface.get_current_texture().map_err(|err| { + error!("Error getting camera renderer surface texture: {err:?}") + }) { + let output_width = 1280; + let output_height = (1280.0 / aspect_ratio) as u32; + + let resampler_frame = resampler_frame + .get_or_init((output_width, output_height), frame::Video::empty); - Ok((window_width as u32, window_height as u32)) + scaler.cached( + frame.frame.format(), + frame.frame.width(), + frame.frame.height(), + format::Pixel::RGBA, + output_width, + output_height, + ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR, + ); + + if let Err(err) = scaler.run(&frame.frame, resampler_frame) { + error!("Error rescaling frame with ffmpeg: {err:?}"); + continue; + } + + self.texture + .get_or_init((output_width, output_height), || { + PreparedTexture::init( + self.device.clone(), + self.queue.clone(), + &self.sampler, + &self.bind_group_layout, + self.uniform_bind_group.clone(), + self.render_pipeline.clone(), + output_width, + output_height, + ) + }) + .render( + &surface, + resampler_frame.data(0), + resampler_frame.stride(0) as u32, + ); + surface.present(); + } + } + Err(ReconfigureEvent::WindowSize(width, height)) => { + self.reconfigure_gpu_surface(width, height); + } + Err(ReconfigureEvent::State(state)) => { + // Aspect ratio is hardcoded until we can derive it from the camera feed + // if !seen_first_frame { // TODO + // self.sync_aspect_ratio_uniforms( + // if state.shape == CameraPreviewShape::Full { + // 16.0 / 9.0 + // } else { + // 1.0 + // }, + // ); + // } + + self.update_state_uniforms(&state); + if let Some(aspect_ratio) = self.aspect_ratio.get_latest_key() { + if let Ok((width, height)) = resize_window(&window, &state, *aspect_ratio) + .map_err(|err| error!("Error resizing camera preview window: {err}")) + { + self.reconfigure_gpu_surface(width, height); + } + } + } + Err(ReconfigureEvent::Shutdown) => return, + } + } + + info!("Camera feed completed. Closing preview window..."); + window.close().ok(); + self.device.destroy(); } /// Reconfigure the GPU surface if the window has changed size - fn reconfigure_gpu_surface( - &mut self, - window_width: u32, - window_height: u32, - ) -> tauri::Result<()> { - self.surface_size - .get_or_init((window_width, window_height), || { - self.surface_config.width = if window_width > 0 { - window_width * GPU_SURFACE_SCALE - } else { - 1 - }; - self.surface_config.height = if window_height > 0 { - window_height * GPU_SURFACE_SCALE - } else { - 1 - }; - self.surface.configure(&self.device, &self.surface_config); - - let window_uniforms = WindowUniforms { - window_height: window_height as f32, - window_width: window_width as f32, - toolbar_percentage: (TOOLBAR_HEIGHT * GPU_SURFACE_SCALE as f32) - / self.surface_config.height as f32, - _padding: 0.0, - }; - self.queue.write_buffer( - &self.window_uniform_buffer, - 0, - bytemuck::cast_slice(&[window_uniforms]), - ); - }); + fn reconfigure_gpu_surface(&mut self, window_width: u32, window_height: u32) { + self.surface_config.width = if window_width > 0 { + window_width * GPU_SURFACE_SCALE + } else { + 1 + }; + self.surface_config.height = if window_height > 0 { + window_height * GPU_SURFACE_SCALE + } else { + 1 + }; + self.surface.configure(&self.device, &self.surface_config); - Ok(()) + let window_uniforms = WindowUniforms { + window_height: window_height as f32, + window_width: window_width as f32, + toolbar_percentage: (TOOLBAR_HEIGHT * GPU_SURFACE_SCALE as f32) + / self.surface_config.height as f32, + _padding: 0.0, + }; + self.queue.write_buffer( + &self.window_uniform_buffer, + 0, + bytemuck::cast_slice(&[window_uniforms]), + ); } /// Update the uniforms which hold the camera preview state - fn update_state_uniforms(&self) { + fn update_state_uniforms(&self, state: &CameraPreviewState) { let state_uniforms = StateUniforms { - shape: match self.state.shape { + shape: match state.shape { CameraPreviewShape::Round => 0.0, CameraPreviewShape::Square => 1.0, CameraPreviewShape::Full => 2.0, }, - size: match self.state.size { + size: match state.size { CameraPreviewSize::Sm => 0.0, CameraPreviewSize::Lg => 1.0, }, - mirrored: if self.state.mirrored { 1.0 } else { 0.0 }, + mirrored: if state.mirrored { 1.0 } else { 0.0 }, _padding: 0.0, }; self.queue.write_buffer( - &self.uniform_buffer, + &self.state_uniform_buffer, 0, bytemuck::cast_slice(&[state_uniforms]), ); } - /// Update the uniforms which hold the camera aspect ratio - fn update_camera_aspect_ratio_uniforms(&self, camera_aspect_ratio: f32) { - let camera_uniforms = CameraUniforms { - camera_aspect_ratio, - _padding: 0.0, - }; - self.queue.write_buffer( - &self.camera_uniform_buffer, - 0, - bytemuck::cast_slice(&[camera_uniforms]), - ); + /// Update the uniforms which hold the camera aspect ratio if it's changed + fn sync_aspect_ratio_uniforms(&mut self, aspect_ratio: f32) { + if self.aspect_ratio.update_key_and_should_init(aspect_ratio) { + let camera_uniforms = CameraUniforms { + camera_aspect_ratio: aspect_ratio, + _padding: 0.0, + }; + self.queue.write_buffer( + &self.camera_uniform_buffer, + 0, + bytemuck::cast_slice(&[camera_uniforms]), + ); + } } } +/// Resize the OS window to the correct size, +/// based on configuration +fn resize_window( + window: &WebviewWindow, + state: &CameraPreviewState, + aspect: f32, +) -> tauri::Result<(u32, u32)> { + let base: f32 = if state.size == CameraPreviewSize::Sm { + 230.0 + } else { + 400.0 + }; + let window_width = if state.shape == CameraPreviewShape::Full { + if aspect >= 1.0 { base * aspect } else { base } + } else { + base + }; + let window_height = if state.shape == CameraPreviewShape::Full { + if aspect >= 1.0 { base } else { base / aspect } + } else { + base + } + TOOLBAR_HEIGHT; + + let (monitor_size, monitor_offset, monitor_scale_factor): ( + PhysicalSize, + LogicalPosition, + _, + ) = if let Some(monitor) = window.current_monitor()? { + let size = monitor.position().to_logical(monitor.scale_factor()); + (*monitor.size(), size, monitor.scale_factor()) + } else { + (PhysicalSize::new(640, 360), LogicalPosition::new(0, 0), 1.0) + }; + + let x = (monitor_size.width as f64 / monitor_scale_factor - window_width as f64 - 100.0) as u32 + + monitor_offset.x; + let y = (monitor_size.height as f64 / monitor_scale_factor - window_height as f64 - 100.0) + as u32 + + monitor_offset.y; + + window.set_size(LogicalSize::new(window_width, window_height))?; + window.set_position(LogicalPosition::new(x, y))?; + + Ok((window_width as u32, window_height as u32)) +} + fn render_solid_frame(color: [u8; 4], width: u32, height: u32) -> (Vec, u32) { let pixel_count = (height * width) as usize; let buffer: Vec = color @@ -717,7 +723,6 @@ fn render_solid_frame(color: [u8; 4], width: u32, height: u32) -> (Vec, u32) pub struct PreparedTexture { texture: wgpu::Texture, - texture_view: wgpu::TextureView, bind_group: wgpu::BindGroup, uniform_bind_group: wgpu::BindGroup, render_pipeline: wgpu::RenderPipeline, @@ -772,7 +777,6 @@ impl PreparedTexture { Self { texture, - texture_view, bind_group, uniform_bind_group, render_pipeline, @@ -861,6 +865,10 @@ impl Cached { &mut self.value.as_mut().expect("checked above").1 } + + pub fn get_latest_key(&self) -> Option<&K> { + self.value.as_ref().map(|(k, _)| k) + } } impl Cached { diff --git a/apps/desktop/src-tauri/src/deeplink_actions.rs b/apps/desktop/src-tauri/src/deeplink_actions.rs index c6e2f9296b..ca6b9c100a 100644 --- a/apps/desktop/src-tauri/src/deeplink_actions.rs +++ b/apps/desktop/src-tauri/src/deeplink_actions.rs @@ -4,9 +4,7 @@ use std::path::{Path, PathBuf}; use tauri::{AppHandle, Manager, Url}; use tracing::trace; -use crate::{ - App, ArcLock, camera::CameraPreview, recording::StartRecordingInputs, windows::ShowCapWindow, -}; +use crate::{App, ArcLock, recording::StartRecordingInputs, windows::ShowCapWindow}; #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -115,9 +113,8 @@ impl DeepLinkAction { mode, } => { let state = app.state::>(); - let camera_preview = app.state::(); - crate::set_camera_input(app.clone(), state.clone(), camera_preview, camera).await?; + crate::set_camera_input(app.clone(), state.clone(), camera).await?; crate::set_mic_input(state.clone(), mic_label).await?; let capture_target: ScreenCaptureTarget = match capture_mode { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index d5e88dd03a..dd3ad0133b 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -25,7 +25,7 @@ mod windows; use audio::AppSounds; use auth::{AuthStore, AuthenticationInvalid, Plan}; -use camera::{CameraPreview, CameraWindowState}; +use camera::CameraPreviewState; use cap_displays::{DisplayId, WindowId, bounds::LogicalBounds}; use cap_editor::EditorInstance; use cap_editor::EditorState; @@ -62,7 +62,6 @@ use serde_json::json; use specta::Type; use std::collections::BTreeMap; use std::path::Path; -use std::time::Duration; use std::{ fs::File, future::Future, @@ -84,7 +83,6 @@ use tauri_plugin_shell::ShellExt; use tauri_specta::Event; use tokio::sync::mpsc; use tokio::sync::{Mutex, RwLock}; -use tokio::time::timeout; use tracing::debug; use tracing::error; use tracing::trace; @@ -94,6 +92,7 @@ use windows::EditorWindowIds; use windows::set_window_transparent; use windows::{CapWindowId, ShowCapWindow}; +use crate::camera::CameraPreviewManager; use crate::upload::build_video_meta; #[allow(clippy::large_enum_variant)] @@ -114,6 +113,8 @@ pub struct App { #[serde(skip)] camera_feed: Option>>, #[serde(skip)] + camera_preview: CameraPreviewManager, + #[serde(skip)] camera_feed_initialization: Option>, #[serde(skip)] mic_feed: Option, @@ -255,7 +256,6 @@ async fn set_mic_input(state: MutableState<'_, App>, label: Option) -> R async fn set_camera_input( app_handle: AppHandle, state: MutableState<'_, App>, - camera_preview: State<'_, CameraPreview>, id: Option, ) -> Result { let mut app = state.write().await; @@ -285,7 +285,7 @@ async fn set_camera_input( app.camera_feed_initialization = Some(shutdown_tx); } - let window = ShowCapWindow::Camera.show(&app_handle).await.unwrap(); + ShowCapWindow::Camera.show(&app_handle).await.unwrap(); if let Some(win) = CapWindowId::Main.get(&app_handle) { win.set_focus().ok(); }; @@ -295,32 +295,11 @@ async fn set_camera_input( .and_then(|v| v.map(|v| v.enable_native_camera_preview)) .unwrap_or_default() { - let (camera_tx, camera_rx) = flume::bounded::(4); - - let prev_err = &mut None; - if timeout(Duration::from_secs(3), async { - while let Err(err) = camera_preview - .init_preview_window(window.clone(), camera_rx.clone()) - .await - { - error!("Error initializing camera feed: {err}"); - *prev_err = Some(err); - tokio::time::sleep(Duration::from_millis(200)).await; - } - }) - .await - .is_err() - { - let _ = window.close(); - return Err(format!("Timeout initializing camera preview: {prev_err:?}")); - }; - - Some(camera_tx) + app.camera_preview.attach() } else { - None + app.camera_tx.clone() }; - let legacy_camera_tx = app.camera_tx.clone(); drop(app); let fut = CameraFeed::init(id); @@ -335,11 +314,7 @@ async fn set_camera_input( } if app.camera_feed.is_none() { - if let Some(camera_tx) = camera_tx { - feed.attach(camera_tx); - } else { - feed.attach(legacy_camera_tx); - } + feed.attach(camera_tx); app.camera_feed = Some(Arc::new(Mutex::new(feed))); Ok(true) } else { @@ -356,8 +331,8 @@ async fn set_camera_input( cancel.send(()).await.ok(); } app.camera_feed.take(); - if let Some(w) = CapWindowId::Camera.get(&app_handle) { - w.close().ok(); + if let Some(win) = CapWindowId::Camera.get(&app_handle) { + win.close().ok(); } Ok(true) } @@ -1871,19 +1846,21 @@ async fn set_server_url(app: MutableState<'_, App>, server_url: String) -> Resul #[tauri::command] #[specta::specta] async fn set_camera_preview_state( - store: State<'_, CameraPreview>, - state: CameraWindowState, -) -> Result<(), ()> { - store.save(&state).map_err(|err| { - error!("Error saving camera window state: {err}"); - })?; + app: MutableState<'_, App>, + state: CameraPreviewState, +) -> Result<(), String> { + app.read() + .await + .camera_preview + .set_state(state) + .map_err(|err| format!("Error saving camera window state: {err}"))?; Ok(()) } #[tauri::command] #[specta::specta] -async fn await_camera_preview_ready(store: State<'_, CameraPreview>) -> Result { +async fn await_camera_preview_ready(app: MutableState<'_, App>) -> Result { // store.wait_for_camera_to_load().await; // TODO: Reimplement this Ok(true) } @@ -2121,6 +2098,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { handle: app.clone(), camera_feed: None, camera_feed_initialization: None, + camera_preview: CameraPreviewManager::new(&app), mic_samples_tx: audio_input_tx, mic_feed: None, recording_state: RecordingState::None, @@ -2136,12 +2114,6 @@ pub async fn run(recording_logging_handle: LoggingHandle) { }), }))); - if let Ok(s) = CameraPreview::init(&app) - .map_err(|err| error!("Error initializing camera preview: {err}")) - { - app.manage(s); - } - app.manage(Arc::new(RwLock::new( ClipboardContext::new().expect("Failed to create clipboard context"), ))); @@ -2209,16 +2181,12 @@ pub async fn run(recording_logging_handle: LoggingHandle) { CapWindowId::Main => { let app = app.clone(); tokio::spawn(async move { - let state = app.state::>>(); + let state = app.state::>(); let app_state = &mut *state.write().await; if !app_state.is_recording_active_or_pending() { app_state.mic_feed.take(); app_state.camera_feed.take(); - - if let Some(camera) = CapWindowId::Camera.get(&app) { - let _ = camera.close(); - } } }); } @@ -2240,6 +2208,16 @@ pub async fn run(recording_logging_handle: LoggingHandle) { app.state::() .destroy(&display_id, app.global_shortcut()); } + CapWindowId::Camera => { + let app = app.clone(); + tokio::spawn(async move { + app.state::>() + .write() + .await + .camera_preview + .on_window_close(); + }); + } _ => {} }; } @@ -2311,15 +2289,17 @@ pub async fn run(recording_logging_handle: LoggingHandle) { } tauri::RunEvent::WindowEvent { event: WindowEvent::Resized(size), - label, .. } => { - if let Some(window) = handle.get_webview_window(&label) { - let size = size.to_logical(window.scale_factor().unwrap_or(1.0)); + let handle = handle.clone(); + tokio::spawn(async move { handle - .state::() - .update_window_size(size.width, size.height); - } + .state::>() + .read() + .await + .camera_preview + .on_window_resize(size.width, size.height) + }); } _ => {} }); diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 918d156bd1..77ffea1172 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -646,8 +646,11 @@ async fn handle_recording_end( if let Some(v) = CapWindowId::Camera.get(&handle) { let _ = v.close(); } - app.camera_feed.take(); app.mic_feed.take(); + app.camera_feed.take(); + if let Some(win) = CapWindowId::Camera.get(&handle) { + win.close().ok(); + } } CurrentRecordingChanged.emit(&handle).ok(); diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index d059271b68..bf595a2e35 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -1,6 +1,7 @@ #![allow(unused_mut)] #![allow(unused_imports)] +use anyhow::anyhow; use cap_displays::{Display, DisplayId}; use futures::pin_mut; use serde::Deserialize; @@ -16,7 +17,7 @@ use tauri::{ WebviewWindow, WebviewWindowBuilder, Wry, }; use tokio::sync::RwLock; -use tracing::debug; +use tracing::{debug, error}; use crate::{ App, ArcLock, fake_window, @@ -359,47 +360,75 @@ impl ShowCapWindow { Self::Camera => { const WINDOW_SIZE: f64 = 230.0 * 2.0; - let port = app.state::>>().read().await.camera_ws_port; - let mut window_builder = self - .window_builder(app, "/camera") - .maximized(false) - .resizable(false) - .shadow(false) - .fullscreen(false) - .always_on_top(true) - .visible_on_all_workspaces(true) - .skip_taskbar(true) - .position( - 100.0, - (monitor.size().height as f64) / monitor.scale_factor() - - WINDOW_SIZE - - 100.0, - ) - .initialization_script(format!( - " + let enable_native_camera_preview = GeneralSettingsStore::get(&app) + .ok() + .and_then(|v| v.map(|v| v.enable_native_camera_preview)) + .unwrap_or_default(); + + { + let state = app.state::>(); + let mut state = state.write().await; + + if enable_native_camera_preview && state.camera_preview.is_initialized() { + error!("Unable to initialize camera preview as one already exists!"); + if let Some(window) = CapWindowId::Camera.get(&app) { + window.show().ok(); + } + return Err(anyhow!( + "Unable to initialize camera preview as one already exists!" + ) + .into()); + } + + let mut window_builder = self + .window_builder(app, "/camera") + .maximized(false) + .resizable(false) + .shadow(false) + .fullscreen(false) + .always_on_top(true) + .visible_on_all_workspaces(true) + .skip_taskbar(true) + .position( + 100.0, + (monitor.size().height as f64) / monitor.scale_factor() + - WINDOW_SIZE + - 100.0, + ) + .initialization_script(format!( + " window.__CAP__ = window.__CAP__ ?? {{}}; - window.__CAP__.cameraWsPort = {port}; + window.__CAP__.cameraWsPort = {}; ", - )) - .transparent(true) - .visible(false); // We set this true in `CameraWindowState::init_window` + state.camera_ws_port + )) + .transparent(true) + .visible(false); // We set this true in `CameraWindowState::init_window` - let window = window_builder.build()?; + let window = window_builder.build()?; - #[cfg(target_os = "macos")] - { - _ = window.run_on_main_thread({ - let window = window.as_ref().window(); - move || unsafe { - let win = window.ns_window().unwrap() as *const objc2_app_kit::NSWindow; - (*win).setCollectionBehavior( - (*win).collectionBehavior() | objc2_app_kit::NSWindowCollectionBehavior::FullScreenAuxiliary, - ); + if enable_native_camera_preview { + if let Err(err) = state.camera_preview.init_window(window.clone()).await { + error!("Error initializing camera preview: {err}"); + window.close().ok(); } - }); - } + } - window + #[cfg(target_os = "macos")] + { + _ = window.run_on_main_thread({ + let window = window.as_ref().window(); + move || unsafe { + let win = window.ns_window().unwrap() as *const objc2_app_kit::NSWindow; + (*win).setCollectionBehavior( + (*win).collectionBehavior() | objc2_app_kit::NSWindowCollectionBehavior::FullScreenAuxiliary, + ); + } + }); + } + + window + } } Self::WindowCaptureOccluder { screen_id } => { let Some(display) = Display::from_id(screen_id) else { diff --git a/apps/desktop/src/routes/camera.tsx b/apps/desktop/src/routes/camera.tsx index e0f9c075b0..c1d7e9f4b5 100644 --- a/apps/desktop/src/routes/camera.tsx +++ b/apps/desktop/src/routes/camera.tsx @@ -58,8 +58,6 @@ export default function () { } function NativeCameraPreviewPage() { - const { rawOptions } = useRecordingOptions(); - const [state, setState] = makePersisted( createStore({ size: "sm", From fc4a6ab4e17b5026ac6d8c29226b7f1f2d9fa939 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 25 Aug 2025 23:17:57 +0800 Subject: [PATCH 05/18] rename camera feed --- crates/camera/src/lib.rs | 42 ++++++++++++++++++++---------------- crates/camera/src/macos.rs | 2 +- crates/camera/src/windows.rs | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index a15803521f..a1af62c6f3 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -210,28 +210,34 @@ impl CameraInfo { &self, format: Format, callback: impl FnMut(CapturedFrame) + 'static, - ) -> Result { - #[cfg(target_os = "macos")] - { - Ok(RecordingHandle { - native: start_capturing_impl(self, format, Box::new(callback))?, - }) - } - #[cfg(windows)] - { - Ok(RecordingHandle { - native: start_capturing_impl(self, format, Box::new(callback))?, - }) - } + ) -> Result { + Ok(CaptureHandle { + #[cfg(target_os = "macos")] + native: Some(start_capturing_impl(self, format, Box::new(callback))?), + #[cfg(windows)] + native: Some(start_capturing_impl(self, format, Box::new(callback))?), + }) } } -pub struct RecordingHandle { - native: NativeRecordingHandle, +#[must_use = "must be held for the duration of the recording"] +pub struct CaptureHandle { + native: Option, } -impl RecordingHandle { - pub fn stop_capturing(self) -> Result<(), String> { - self.native.stop_capturing() +impl CaptureHandle { + pub fn stop_capturing(mut self) -> Result<(), String> { + if let Some(feed) = self.native.take() { + feed.stop_capturing()?; + } + Ok(()) + } +} + +impl Drop for CaptureHandle { + fn drop(&mut self) { + if let Some(feed) = self.native.take() { + feed.stop_capturing().ok(); + } } } diff --git a/crates/camera/src/macos.rs b/crates/camera/src/macos.rs index 715a8f5784..bf81e186e3 100644 --- a/crates/camera/src/macos.rs +++ b/crates/camera/src/macos.rs @@ -68,7 +68,7 @@ impl ModelID { pub type NativeFormat = arc::R; -pub type NativeRecordingHandle = AVFoundationRecordingHandle; +pub type NativeCaptureHandle = AVFoundationRecordingHandle; fn find_device(info: &CameraInfo) -> Option> { let devices = list_video_devices(); diff --git a/crates/camera/src/windows.rs b/crates/camera/src/windows.rs index 4c2a6bdf88..9b7771efde 100644 --- a/crates/camera/src/windows.rs +++ b/crates/camera/src/windows.rs @@ -61,7 +61,7 @@ impl ModelID { #[derive(Debug)] pub struct NativeCapturedFrame(cap_camera_windows::Frame); -pub type NativeRecordingHandle = WindowsCaptureHandle; +pub type NativeCaptureHandle = WindowsCaptureHandle; pub(super) fn start_capturing_impl( camera: &CameraInfo, From b03a60ee1e299d4d22d0accb2ea4c1c092800848 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 26 Aug 2025 03:50:29 +0800 Subject: [PATCH 06/18] clippy --- crates/cpal-ffmpeg/src/lib.rs | 2 +- crates/cursor-capture/src/position.rs | 4 ++-- crates/rendering/src/layout.rs | 34 +++++++++++++-------------- crates/scap-ffmpeg/src/cpal.rs | 2 +- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/crates/cpal-ffmpeg/src/lib.rs b/crates/cpal-ffmpeg/src/lib.rs index 0074e82591..2b0a46eef2 100644 --- a/crates/cpal-ffmpeg/src/lib.rs +++ b/crates/cpal-ffmpeg/src/lib.rs @@ -27,7 +27,7 @@ impl DataExt for ::cpal::Data { if matches!(format_typ, sample::Type::Planar) { for i in 0..config.channels { - let plane_size = sample_count * sample_size as usize; + let plane_size = sample_count * sample_size; let base = (i as usize) * plane_size; ffmpeg_frame diff --git a/crates/cursor-capture/src/position.rs b/crates/cursor-capture/src/position.rs index b41248784c..58ceb3a0aa 100644 --- a/crates/cursor-capture/src/position.rs +++ b/crates/cursor-capture/src/position.rs @@ -49,11 +49,11 @@ impl RelativeCursorPosition { { let logical_bounds = display.raw_handle().logical_bounds()?; - return Some(Self { + Some(Self { x: raw.x - logical_bounds.position().x() as i32, y: raw.y - logical_bounds.position().y() as i32, display, - }); + }) } } diff --git a/crates/rendering/src/layout.rs b/crates/rendering/src/layout.rs index 03e6f1bfd5..0e0eb5745b 100644 --- a/crates/rendering/src/layout.rs +++ b/crates/rendering/src/layout.rs @@ -73,65 +73,63 @@ impl InterpolatedLayout { if cursor.time < segment.start && cursor.time >= transition_start { let prev_mode = cursor .prev_segment - .map(|s| s.mode.clone()) + .map(|s| s.mode) .unwrap_or(LayoutMode::Default); let progress = (cursor.time - transition_start) / LAYOUT_TRANSITION_DURATION; ( prev_mode, - segment.mode.clone(), + segment.mode, ease_in_out(progress as f32) as f64, ) } else if cursor.time >= transition_end && cursor.time < segment.end { if let Some(next_seg) = cursor.next_segment() { let progress = (cursor.time - transition_end) / LAYOUT_TRANSITION_DURATION; ( - segment.mode.clone(), - next_seg.mode.clone(), + segment.mode, + next_seg.mode, ease_in_out(progress as f32) as f64, ) } else { let progress = (cursor.time - transition_end) / LAYOUT_TRANSITION_DURATION; ( - segment.mode.clone(), + segment.mode, LayoutMode::Default, ease_in_out(progress as f32) as f64, ) } } else { - (segment.mode.clone(), segment.mode.clone(), 1.0) + (segment.mode, segment.mode, 1.0) } } else if let Some(next_segment) = cursor.next_segment() { let transition_start = next_segment.start - LAYOUT_TRANSITION_DURATION; if cursor.time >= transition_start { let prev_mode = cursor .prev_segment - .map(|s| s.mode.clone()) + .map(|s| s.mode) .unwrap_or(LayoutMode::Default); let progress = (cursor.time - transition_start) / LAYOUT_TRANSITION_DURATION; ( prev_mode, - next_segment.mode.clone(), + next_segment.mode, ease_in_out(progress as f32) as f64, ) } else if let Some(prev_segment) = cursor.prev_segment { if cursor.time < prev_segment.end + 0.05 { - (prev_segment.mode.clone(), LayoutMode::Default, 1.0) + (prev_segment.mode, LayoutMode::Default, 1.0) } else { (LayoutMode::Default, LayoutMode::Default, 1.0) } } else { (LayoutMode::Default, LayoutMode::Default, 1.0) } - } else { - if let Some(prev_segment) = cursor.prev_segment { - if cursor.time < prev_segment.end + 0.05 { - (prev_segment.mode.clone(), LayoutMode::Default, 1.0) - } else { - (LayoutMode::Default, LayoutMode::Default, 1.0) - } + } else if let Some(prev_segment) = cursor.prev_segment { + if cursor.time < prev_segment.end + 0.05 { + (prev_segment.mode, LayoutMode::Default, 1.0) } else { (LayoutMode::Default, LayoutMode::Default, 1.0) } + } else { + (LayoutMode::Default, LayoutMode::Default, 1.0) }; let (start_camera_opacity, start_screen_opacity, start_camera_scale) = @@ -198,9 +196,9 @@ impl InterpolatedLayout { screen_opacity, camera_scale, layout_mode: if transition_progress > 0.5 { - next_mode.clone() + next_mode } else { - current_mode.clone() + current_mode }, transition_progress, from_mode: current_mode, diff --git a/crates/scap-ffmpeg/src/cpal.rs b/crates/scap-ffmpeg/src/cpal.rs index c3afb79ff6..cabbfc0921 100644 --- a/crates/scap-ffmpeg/src/cpal.rs +++ b/crates/scap-ffmpeg/src/cpal.rs @@ -27,7 +27,7 @@ impl DataExt for ::cpal::Data { if matches!(format_typ, sample::Type::Planar) { for i in 0..config.channels { - let plane_size = sample_count * sample_size as usize; + let plane_size = sample_count * sample_size; let base = (i as usize) * plane_size; ffmpeg_frame From ebb28ecb2510d7da39c7a17504352f49e7be81d7 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 03:52:46 +0800 Subject: [PATCH 07/18] Camera feed actor (#946) * wip * vibe code it * wip * make recording work * handle InputConnected in connecting Lock --------- Co-authored-by: Brendan Allan --- apps/desktop/src-tauri/src/camera.rs | 2 +- apps/desktop/src-tauri/src/camera_legacy.rs | 2 +- .../desktop/src-tauri/src/deeplink_actions.rs | 4 +- apps/desktop/src-tauri/src/hotkeys.rs | 2 +- apps/desktop/src-tauri/src/lib.rs | 128 ++-- apps/desktop/src-tauri/src/recording.rs | 14 +- crates/recording/src/feeds/camera.rs | 671 +++++++++++------- crates/recording/src/feeds/mod.rs | 4 +- crates/recording/src/lib.rs | 3 + crates/recording/src/sources/audio_input.rs | 2 +- crates/recording/src/sources/camera.rs | 22 +- crates/recording/src/studio_recording.rs | 46 +- 12 files changed, 485 insertions(+), 415 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index bd4f2524e9..540b9119d5 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -1,5 +1,5 @@ use anyhow::{Context, anyhow}; -use cap_recording::feeds::RawCameraFrame; +use cap_recording::feeds::camera::RawCameraFrame; use ffmpeg::{ format::{self, Pixel}, frame, diff --git a/apps/desktop/src-tauri/src/camera_legacy.rs b/apps/desktop/src-tauri/src/camera_legacy.rs index bb1afa70d5..2aeaeaf3b0 100644 --- a/apps/desktop/src-tauri/src/camera_legacy.rs +++ b/apps/desktop/src-tauri/src/camera_legacy.rs @@ -1,4 +1,4 @@ -use cap_recording::feeds::RawCameraFrame; +use cap_recording::feeds::camera::RawCameraFrame; use flume::Sender; use tokio_util::sync::CancellationToken; diff --git a/apps/desktop/src-tauri/src/deeplink_actions.rs b/apps/desktop/src-tauri/src/deeplink_actions.rs index ca6b9c100a..a9e2b080dc 100644 --- a/apps/desktop/src-tauri/src/deeplink_actions.rs +++ b/apps/desktop/src-tauri/src/deeplink_actions.rs @@ -1,4 +1,4 @@ -use cap_recording::{RecordingMode, feeds::DeviceOrModelID, sources::ScreenCaptureTarget}; +use cap_recording::{RecordingMode, feeds::camera::DeviceOrModelID, sources::ScreenCaptureTarget}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tauri::{AppHandle, Manager, Url}; @@ -114,7 +114,7 @@ impl DeepLinkAction { } => { let state = app.state::>(); - crate::set_camera_input(app.clone(), state.clone(), camera).await?; + crate::set_camera_input(state.clone(), camera).await?; crate::set_mic_input(state.clone(), mic_label).await?; let capture_target: ScreenCaptureTarget = match capture_mode { diff --git a/apps/desktop/src-tauri/src/hotkeys.rs b/apps/desktop/src-tauri/src/hotkeys.rs index d59cab3f3c..6d16f9afa3 100644 --- a/apps/desktop/src-tauri/src/hotkeys.rs +++ b/apps/desktop/src-tauri/src/hotkeys.rs @@ -101,7 +101,7 @@ pub fn init(app: &AppHandle) { ) .unwrap(); - let mut store = match HotkeysStore::get(app) { + let store = match HotkeysStore::get(app) { Ok(Some(s)) => s, Ok(None) => HotkeysStore::default(), Err(e) => { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index d28bd68fae..2176ef92fb 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -34,7 +34,8 @@ use cap_project::{ }; use cap_recording::{ feeds::{ - self, CameraFeed, DeviceOrModelID, RawCameraFrame, + self, + camera::{CameraFeed, DeviceOrModelID, RawCameraFrame}, microphone::{self, MicrophoneFeed}, }, sources::ScreenCaptureTarget, @@ -67,7 +68,6 @@ use std::{ process::Command, str::FromStr, sync::Arc, - time::Duration, }; use tauri::{AppHandle, Manager, State, Window, WindowEvent}; use tauri_plugin_deep_link::DeepLinkExt; @@ -77,10 +77,7 @@ use tauri_plugin_notification::{NotificationExt, PermissionState}; use tauri_plugin_opener::OpenerExt; use tauri_plugin_shell::ShellExt; use tauri_specta::Event; -use tokio::{ - sync::{Mutex, RwLock, mpsc}, - time::timeout, -}; +use tokio::sync::RwLock; use tracing::{error, trace}; use upload::{S3UploadMeta, create_or_get_video, upload_image, upload_video}; use web_api::ManagerExt as WebManagerExt; @@ -105,12 +102,8 @@ pub struct App { #[deprecated = "can be removed when native camera preview is ready"] camera_ws_port: u16, #[serde(skip)] - camera_feed: Option>>, - #[serde(skip)] camera_preview: CameraPreviewManager, #[serde(skip)] - camera_feed_initialization: Option>, - #[serde(skip)] handle: AppHandle, #[serde(skip)] recording_state: RecordingState, @@ -118,6 +111,8 @@ pub struct App { recording_logging_handle: LoggingHandle, #[serde(skip)] mic_feed: ActorRef, + #[serde(skip)] + camera_feed: ActorRef, server_url: String, } @@ -245,89 +240,29 @@ async fn set_mic_input(state: MutableState<'_, App>, label: Option) -> R #[tauri::command] #[specta::specta] async fn set_camera_input( - app_handle: AppHandle, state: MutableState<'_, App>, id: Option, -) -> Result { - let mut app = state.write().await; +) -> Result<(), String> { + let camera_feed = state.read().await.camera_feed.clone(); - match (id, app.camera_feed.as_ref()) { - (Some(id), Some(camera_feed)) => { + match id { + None => { camera_feed - .lock() - .await - .switch_cameras(id) + .ask(feeds::camera::RemoveInput) .await .map_err(|e| e.to_string())?; - Ok(true) - } - (Some(id), None) => { - let (shutdown_tx, mut shutdown_rx) = mpsc::channel(1); - if let Some(cancel) = app.camera_feed_initialization.as_ref() { - // Ask currently running setup to abort - cancel.send(()).await.ok(); - - // We can assume a window was already initialized. - // Stop it so we can recreate it with the correct `camera_tx` - if let Some(win) = CapWindowId::Camera.get(&app_handle) { - win.close().unwrap(); // TODO: Error handling - }; - } else { - app.camera_feed_initialization = Some(shutdown_tx); - } - - ShowCapWindow::Camera.show(&app_handle).await.unwrap(); - if let Some(win) = CapWindowId::Main.get(&app_handle) { - win.set_focus().ok(); - }; - - let camera_tx = if GeneralSettingsStore::get(&app_handle) - .ok() - .and_then(|v| v.map(|v| v.enable_native_camera_preview)) - .unwrap_or_default() - { - app.camera_preview.attach() - } else { - app.camera_tx.clone() - }; - - drop(app); - - let fut = CameraFeed::init(id); - - tokio::select! { - result = fut => { - let feed = result.map_err(|err| err.to_string())?; - let mut app = state.write().await; - - if let Some(cancel) = app.camera_feed_initialization.take() { - cancel.send(()).await.ok(); - } - - if app.camera_feed.is_none() { - feed.attach(camera_tx); - app.camera_feed = Some(Arc::new(Mutex::new(feed))); - Ok(true) - } else { - Ok(false) - } - } - _ = shutdown_rx.recv() => { - Ok(false) - } - } } - (None, _) => { - if let Some(cancel) = app.camera_feed_initialization.take() { - cancel.send(()).await.ok(); - } - app.camera_feed.take(); - if let Some(win) = CapWindowId::Camera.get(&app_handle) { - win.close().ok(); - } - Ok(true) + Some(id) => { + camera_feed + .ask(feeds::camera::SetInput { id }) + .await + .map_err(|e| e.to_string())? + .await + .map_err(|e| e.to_string())?; } } + + Ok(()) } #[derive(specta::Type, Serialize, tauri_specta::Event, Clone)] @@ -1994,6 +1929,23 @@ pub async fn run(recording_logging_handle: LoggingHandle) { let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); + let camera_feed = { + let (error_tx, error_rx) = flume::bounded(1); + + let mic_feed = CameraFeed::spawn(CameraFeed::new(error_tx)); + + // TODO: make this part of a global actor one day + tokio::spawn(async move { + let Ok(err) = error_rx.recv_async().await else { + return; + }; + + error!("Camera feed actor error: {err:?}"); + }); + + mic_feed + }; + let mic_feed = { let (error_tx, error_rx) = flume::bounded(1); @@ -2108,12 +2060,11 @@ pub async fn run(recording_logging_handle: LoggingHandle) { camera_tx, camera_ws_port, handle: app.clone(), - camera_feed: None, - camera_feed_initialization: None, camera_preview: CameraPreviewManager::new(&app), recording_state: RecordingState::None, recording_logging_handle, mic_feed, + camera_feed, server_url: GeneralSettingsStore::get(&app) .ok() .flatten() @@ -2198,7 +2149,10 @@ pub async fn run(recording_logging_handle: LoggingHandle) { if !app_state.is_recording_active_or_pending() { let _ = app_state.mic_feed.ask(microphone::RemoveInput).await; - app_state.camera_feed.take(); + let _ = app_state + .camera_feed + .ask(feeds::camera::RemoveInput) + .await; } }); } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 8be69ed12b..9cfa7498f5 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6,7 +6,7 @@ use cap_project::{ }; use cap_recording::{ CompletedStudioRecording, RecordingError, RecordingMode, StudioRecordingHandle, - feeds::{CameraFeed, microphone}, + feeds::{camera, microphone}, instant_recording::{CompletedInstantRecording, InstantRecordingHandle}, sources::{CaptureDisplay, CaptureWindow, ScreenCaptureTarget, screen_capture}, }; @@ -180,7 +180,7 @@ pub async fn list_capture_windows() -> Vec { #[tauri::command(async)] #[specta::specta] pub fn list_cameras() -> Vec { - CameraFeed::list_cameras() + cap_camera::list_cameras().collect() } #[derive(Deserialize, Type, Clone, Debug)] @@ -359,10 +359,17 @@ pub async fn start_recording( Err(e) => return Err(e.to_string()), }; + let camera_feed = match state.camera_feed.ask(camera::Lock).await { + Ok(lock) => Some(Arc::new(lock)), + Err(SendError::HandlerError(camera::LockFeedError::NoInput)) => None, + Err(e) => return Err(e.to_string()), + }; + let base_inputs = cap_recording::RecordingBaseInputs { capture_target: inputs.capture_target.clone(), capture_system_audio: inputs.capture_system_audio, mic_feed, + camera_feed, }; let (actor, actor_done_rx) = match inputs.mode { @@ -371,7 +378,6 @@ pub async fn start_recording( id.clone(), recording_dir.clone(), base_inputs, - state.camera_feed.clone(), general_settings .map(|s| s.custom_cursor_capture) .unwrap_or_default(), @@ -653,7 +659,7 @@ async fn handle_recording_end( let _ = v.close(); } let _ = app.mic_feed.ask(microphone::RemoveInput).await; - app.camera_feed.take(); + let _ = app.camera_feed.ask(camera::RemoveInput).await; if let Some(win) = CapWindowId::Camera.get(&handle) { win.close().ok(); } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 8b88253d2d..180386f64c 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -1,42 +1,23 @@ -use cap_fail::{fail, fail_err}; +use cap_camera::CameraInfo; +use cap_fail::fail_err; use cap_media_info::VideoInfo; use ffmpeg::frame; -use flume::{Receiver, Sender, TryRecvError, TrySendError}; -use futures::channel::oneshot; +use futures::{FutureExt, future::BoxFuture}; +use kameo::prelude::*; +use replace_with::replace_with_or_abort; use std::{ cmp::Ordering, - sync::mpsc, - thread, + ops::Deref, time::{Duration, Instant}, }; -use tracing::{debug, error, info, trace, warn}; +use tokio::sync::oneshot; +use tracing::{debug, error, trace, warn}; use cap_camera_ffmpeg::*; -pub struct CameraFeedInfo { - pub camera: cap_camera::CameraInfo, - pub video_info: VideoInfo, - pub reference_time: Instant, -} - -#[derive(Debug, thiserror::Error)] -pub enum SwitchCameraError { - #[error("Setup/{0}")] - Setup(#[from] SetupCameraError), - #[error("Failed to send request")] - RequestFailed(oneshot::Canceled), - #[error("Failed to initialize camera")] - InitializeFailed(oneshot::Canceled), -} +type StreamError = (); // TODO: Fix this -enum CameraControl { - Switch( - DeviceOrModelID, - oneshot::Sender>, - ), - AttachConsumer(Sender), - Shutdown, -} +const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); #[derive(Clone)] pub struct RawCameraFrame { @@ -45,271 +26,191 @@ pub struct RawCameraFrame { pub refrence_time: Instant, } -pub struct CameraConnection { - control: Sender, +#[derive(Actor)] +pub struct CameraFeed { + state: State, + senders: Vec>, + input_id_counter: u32, } -impl CameraConnection { - pub fn attach(&self) -> Receiver { - let (sender, receiver) = flume::bounded(60); - self.control - .send(CameraControl::AttachConsumer(sender)) - .ok(); +enum State { + Open(OpenState), + Locked { inner: AttachedState }, +} - receiver +impl State { + fn try_as_open(&mut self) -> Result<&mut OpenState, FeedLockedError> { + if let Self::Open(open_state) = self { + Ok(open_state) + } else { + Err(FeedLockedError) + } } } -#[derive(serde::Serialize, serde::Deserialize, specta::Type, Clone, Debug)] -pub enum DeviceOrModelID { - DeviceID(String), - ModelID(cap_camera::ModelID), +struct OpenState { + connecting: Option, + attached: Option, } -impl DeviceOrModelID { - pub fn from_info(info: &cap_camera::CameraInfo) -> Self { - info.model_id() - .map(|v| Self::ModelID(v.clone())) - .unwrap_or_else(|| Self::DeviceID(info.device_id().to_string())) +impl OpenState { + fn handle_input_connected(&mut self, data: SetupCameraResult, id: DeviceOrModelID) { + if let Some(connecting) = &self.connecting + && id == connecting.id + { + self.attached = Some(AttachedState { + id, + handle: data.handle, + camera_info: data.camera_info, + video_info: data.video_info, + }); + self.connecting = None; + } } } -pub struct CameraFeed { - pub camera_info: cap_camera::CameraInfo, +struct ConnectingState { + id: DeviceOrModelID, + ready: BoxFuture<'static, Result>, +} + +struct AttachedState { + id: DeviceOrModelID, + handle: cap_camera::CaptureHandle, + camera_info: cap_camera::CameraInfo, video_info: VideoInfo, - reference_time: Instant, - control: Sender, } impl CameraFeed { - pub async fn init(selected_camera: DeviceOrModelID) -> Result { - trace!("Initializing camera feed for: {:?}", &selected_camera); - - fail_err!( - "media::feeds::camera::init", - SetupCameraError::Initialisation - ); - - let camera_info = find_camera(&selected_camera).ok_or(SetupCameraError::CameraNotFound)?; - let (control, control_receiver) = flume::bounded(1); - - let (ready_tx, ready_rx) = oneshot::channel(); - - thread::spawn(move || { - run_camera_feed(selected_camera, control_receiver, ready_tx); - }); - - let state = ready_rx - .await - .map_err(|_| SetupCameraError::Initialisation)??; - - let camera_feed = Self { - camera_info, - control, - video_info: state.video_info, - reference_time: state.reference_time, - }; - - Ok(camera_feed) - } - - /// Initialize camera asynchronously, returning a receiver immediately. - /// The actual initialization happens in a background task. - /// Dropping the receiver cancels the initialization. - pub fn init_async( - id: DeviceOrModelID, - ) -> flume::Receiver> { - let (tx, rx) = flume::bounded(1); - - tokio::spawn(async move { - let result = Self::init(id).await; - let _ = tx.send(result); - }); - - rx + pub fn new(error_sender: flume::Sender) -> Self { + Self { + state: State::Open(OpenState { + connecting: None, + attached: None, + }), + senders: Vec::new(), + input_id_counter: 0, + } } +} - pub fn list_cameras() -> Vec { - cap_camera::list_cameras().collect() - } +#[derive(Reply)] +pub struct CameraFeedLock { + actor: ActorRef, + camera_info: cap_camera::CameraInfo, + video_info: VideoInfo, + lock_tx: Recipient, +} - pub fn camera_info(&self) -> cap_camera::CameraInfo { - self.camera_info.clone() +impl CameraFeedLock { + pub fn camera_info(&self) -> &cap_camera::CameraInfo { + &self.camera_info } - pub fn video_info(&self) -> VideoInfo { - self.video_info + pub fn video_info(&self) -> &VideoInfo { + &self.video_info } +} - pub async fn switch_cameras(&mut self, id: DeviceOrModelID) -> Result<(), SwitchCameraError> { - fail_err!( - "media::feeds::camera::switch_cameras", - SwitchCameraError::Setup(SetupCameraError::CameraNotFound) - ); - - let (result_tx, result_rx) = oneshot::channel(); - - let _ = self - .control - .send_async(CameraControl::Switch(id, result_tx)) - .await; - - let data = result_rx - .await - .map_err(SwitchCameraError::RequestFailed)??; - - self.camera_info = data.camera; - self.video_info = data.video_info; - self.reference_time = data.reference_time; +impl Deref for CameraFeedLock { + type Target = ActorRef; - Ok(()) + fn deref(&self) -> &Self::Target { + &self.actor } +} - pub fn create_connection(&self) -> CameraConnection { - CameraConnection { - control: self.control.clone(), - } +impl Drop for CameraFeedLock { + fn drop(&mut self) { + let _ = self.lock_tx.tell(Unlock).blocking_send(); } +} - pub fn attach(&self, sender: Sender) { - self.control - .send(CameraControl::AttachConsumer(sender)) - .ok(); - } +#[derive(serde::Serialize, serde::Deserialize, specta::Type, Clone, Debug, PartialEq)] +pub enum DeviceOrModelID { + DeviceID(String), + ModelID(cap_camera::ModelID), } -impl Drop for CameraFeed { - fn drop(&mut self) { - let _ = self.control.send(CameraControl::Shutdown); +impl DeviceOrModelID { + pub fn from_info(info: &cap_camera::CameraInfo) -> Self { + info.model_id() + .map(|v| Self::ModelID(v.clone())) + .unwrap_or_else(|| Self::DeviceID(info.device_id().to_string())) } } -fn find_camera(selected_camera: &DeviceOrModelID) -> Option { - cap_camera::list_cameras().find(|c| match selected_camera { - DeviceOrModelID::DeviceID(device_id) => c.device_id() == device_id, - DeviceOrModelID::ModelID(model_id) => c.model_id() == Some(model_id), - }) +// Public Requests + +pub struct SetInput { + pub id: DeviceOrModelID, } -// #[tracing::instrument(skip_all)] -fn run_camera_feed( - id: DeviceOrModelID, - control: Receiver, - ready_tx: oneshot::Sender>, -) { - fail!("media::feeds::camera::run panic"); - - let mut senders: Vec> = vec![]; - - let mut state = match setup_camera(id) { - Ok(state) => { - let _ = ready_tx.send(Ok(CameraFeedInfo { - camera: state.camera_info.clone(), - video_info: state.video_info, - reference_time: state.reference_time, - })); - state - } - Err(e) => { - let _ = ready_tx.send(Err(e)); - return; - } - }; +pub struct RemoveInput; - 'outer: loop { - debug!("Video feed camera format: {:#?}", &state.video_info); +pub struct AddSender(pub flume::Sender); - loop { - match control.try_recv() { - Err(TryRecvError::Disconnected) => { - trace!("Control disconnected"); - break 'outer; - } - Ok(CameraControl::Shutdown) => { - state - .handle - .stop_capturing() - .map_err(|err| error!("Error stopping capture: {err:?}")) - .ok(); - println!("Deliberate shutdown"); - break 'outer; - } - Err(TryRecvError::Empty) => {} - Ok(CameraControl::AttachConsumer(sender)) => { - senders.push(sender); - } - Ok(CameraControl::Switch(new_id, switch_result)) => match setup_camera(new_id) { - Ok(new_state) => { - let _ = switch_result.send(Ok(CameraFeedInfo { - camera: new_state.camera_info.clone(), - video_info: new_state.video_info, - reference_time: new_state.reference_time, - })); - state = new_state; - - break; - } - Err(e) => { - let _ = switch_result.send(Err(e)); - continue; - } - }, - } +pub struct Lock; - let Ok(frame) = state.frame_rx.recv_timeout(Duration::from_secs(5)) else { - return; - }; +// Private Events - let mut to_remove = vec![]; +#[derive(Clone)] +struct InputConnected; - for (i, sender) in senders.iter().enumerate() { - if let Err(TrySendError::Disconnected(_)) = sender.try_send(frame.clone()) { - warn!("Camera sender {} disconnected, will be removed", i); - to_remove.push(i); - }; - } +struct InputConnectFailed { + id: DeviceOrModelID, +} - if !to_remove.is_empty() { - // debug!("Removing {} disconnected audio senders", to_remove.len()); - for i in to_remove.into_iter().rev() { - senders.swap_remove(i); - } - } - } - } +struct NewFrame(RawCameraFrame); - info!("Camera feed stopping"); -} +struct Unlock; + +// Impls + +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("FeedLocked")] +pub struct FeedLockedError; -#[derive(Debug, thiserror::Error)] -pub enum SetupCameraError { - #[error("Camera not found")] - CameraNotFound, - #[error("Invalid format")] +#[derive(Clone, Debug, thiserror::Error)] +pub enum SetInputError { + #[error(transparent)] + Locked(#[from] FeedLockedError), + #[error("DeviceNotFound")] + DeviceNotFound, + #[error("BuildStreamCrashed")] + BuildStreamCrashed, // TODO: Maybe rename this? + #[error("InvalidFormat")] InvalidFormat, - #[error("Camera timed out")] - Timeout(mpsc::RecvTimeoutError), + #[error("CameraTimeout")] + Timeout(String), #[error("StartCapturing/{0}")] - StartCapturing(#[from] cap_camera::StartCapturingError), + StartCapturing(String), #[error("Failed to initialize camera")] Initialisation, } -const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); +fn find_camera(selected_camera: &DeviceOrModelID) -> Option { + cap_camera::list_cameras().find(|c| match selected_camera { + DeviceOrModelID::DeviceID(device_id) => c.device_id() == device_id, + DeviceOrModelID::ModelID(model_id) => c.model_id() == Some(model_id), + }) +} -struct SetupCameraState { +struct SetupCameraResult { handle: cap_camera::CaptureHandle, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, - frame_rx: mpsc::Receiver, - reference_time: Instant, + // frame_rx: mpsc::Receiver, } -fn setup_camera(id: DeviceOrModelID) -> Result { - let camera = find_camera(&id).ok_or(SetupCameraError::CameraNotFound)?; - let formats = camera.formats().ok_or(SetupCameraError::InvalidFormat)?; +async fn setup_camera( + id: &DeviceOrModelID, + recipient: Recipient, +) -> Result { + let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?; + let formats = camera.formats().ok_or(SetInputError::InvalidFormat)?; if formats.is_empty() { - return Err(SetupCameraError::InvalidFormat); + return Err(SetInputError::InvalidFormat); } let mut ideal_formats = formats @@ -322,11 +223,6 @@ fn setup_camera(id: DeviceOrModelID) -> Result Result for CameraFeed { + type Reply = + Result>, SetInputError>; - let capture_handle = camera.start_capturing(format.clone(), move |frame| { - let Ok(mut ff_frame) = frame.to_ffmpeg() else { - return; + async fn handle(&mut self, msg: SetInput, ctx: &mut Context) -> Self::Reply { + trace!("CameraFeed.SetInput('{:?}')", &msg.id); + + fail_err!( + "media::feeds::camera::set_input", + SetInputError::Initialisation + ); + + let state = self.state.try_as_open()?; + + // let id = self.input_id_counter; + // self.input_id_counter += 1; + + let (internal_ready_tx, internal_ready_rx) = + oneshot::channel::>(); + let (ready_tx, ready_rx) = + oneshot::channel::>(); + + let ready = { + ready_rx + .map(|v| { + v.map_err(|_| SetInputError::BuildStreamCrashed) + .and_then(|v| v) + }) + .shared() }; - ff_frame.set_pts(Some(frame.timestamp.as_micros() as i64)); + state.connecting = Some(ConnectingState { + id: msg.id.clone(), + ready: internal_ready_rx + .map(|v| { + v.map_err(|_| SetInputError::BuildStreamCrashed) + .and_then(|v| v) + }) + .boxed(), + }); - if let Some(signal) = ready_signal.take() { - let video_info = VideoInfo::from_raw_ffmpeg( - ff_frame.format(), - ff_frame.width(), - ff_frame.height(), - frame_rate, - ); + let id = msg.id.clone(); + let actor_ref = ctx.actor_ref(); + let new_frame_recipient = actor_ref.clone().recipient(); + tokio::spawn(async move { + match setup_camera(&id, new_frame_recipient).await { + Ok(r) => { + let _ = ready_tx.send(Ok((r.camera_info.clone(), r.video_info.clone()))); + let _ = internal_ready_tx.send(Ok(r)); - let _ = signal.send((video_info, frame.reference_time)); - } + let _ = actor_ref.ask(InputConnected).await; + } + Err(e) => { + let _ = ready_tx.send(Err(e.clone())); + let _ = internal_ready_tx.send(Err(e)); - let _ = frame_tx.send(RawCameraFrame { - frame: ff_frame, - timestamp: frame.timestamp, - refrence_time: frame.reference_time, + let _ = actor_ref.tell(InputConnectFailed { id }).await; + } + } }); - })?; - let (video_info, reference_time) = ready_rx - .recv_timeout(CAMERA_INIT_TIMEOUT) - .map_err(SetupCameraError::Timeout)?; + Ok(ready.map(|v| v).boxed()) + } +} - Ok(SetupCameraState { - handle: capture_handle, - camera_info: camera, - video_info, - frame_rx, - reference_time, - }) +impl Message for CameraFeed { + type Reply = Result<(), FeedLockedError>; + + async fn handle(&mut self, _: RemoveInput, _: &mut Context) -> Self::Reply { + trace!("CameraFeed.RemoveInput"); + + let state = self.state.try_as_open()?; + + state.connecting = None; + + if let Some(AttachedState { handle, .. }) = state.attached.take() { + let _ = handle.stop_capturing(); + } + + Ok(()) + } +} + +impl Message for CameraFeed { + type Reply = (); + + async fn handle(&mut self, msg: AddSender, _: &mut Context) -> Self::Reply { + self.senders.push(msg.0); + } +} + +impl Message for CameraFeed { + type Reply = (); + + async fn handle(&mut self, msg: NewFrame, _: &mut Context) -> Self::Reply { + let mut to_remove = vec![]; + + for (i, sender) in self.senders.iter().enumerate() { + if let Err(flume::TrySendError::Disconnected(_)) = sender.try_send(msg.0.clone()) { + warn!("Camera sender {} disconnected, will be removed", i); + to_remove.push(i); + }; + } + + if !to_remove.is_empty() { + debug!("Removing {} disconnected camera senders", to_remove.len()); + for i in to_remove.into_iter().rev() { + self.senders.swap_remove(i); + } + } + } +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum LockFeedError { + #[error(transparent)] + Locked(#[from] FeedLockedError), + #[error("NoInput")] + NoInput, + #[error("InitializeFailed/{0}")] + InitializeFailed(#[from] SetInputError), +} + +impl Message for CameraFeed { + type Reply = Result; + + async fn handle(&mut self, _: Lock, ctx: &mut Context) -> Self::Reply { + trace!("CameraFeed.Lock"); + + let state = self.state.try_as_open()?; + + if let Some(connecting) = &mut state.connecting { + let id = connecting.id.clone(); + let ready = &mut connecting.ready; + let data = ready.await?; + + state.handle_input_connected(data, id); + } + + let Some(attached) = state.attached.take() else { + return Err(LockFeedError::NoInput); + }; + + let camera_info = attached.camera_info.clone(); + let video_info = attached.video_info.clone(); + + self.state = State::Locked { inner: attached }; + + Ok(CameraFeedLock { + camera_info, + video_info, + actor: ctx.actor_ref(), + lock_tx: ctx.actor_ref().recipient(), + }) + } +} + +impl Message for CameraFeed { + type Reply = Result<(), FeedLockedError>; + + async fn handle( + &mut self, + _: InputConnected, + _: &mut Context, + ) -> Self::Reply { + trace!("CameraFeed.InputConnected"); + + let state = self.state.try_as_open()?; + + if let Some(connecting) = &mut state.connecting { + let id = connecting.id.clone(); + let ready = &mut connecting.ready; + let res = ready.await; + + if let Ok(data) = res { + println!("connected: {:?}", &id); + state.handle_input_connected(data, id); + } + } + + Ok(()) + } +} + +impl Message for CameraFeed { + type Reply = Result<(), FeedLockedError>; + + async fn handle( + &mut self, + msg: InputConnectFailed, + _: &mut Context, + ) -> Self::Reply { + trace!("CameraFeed.InputConnectFailed"); + + let state = self.state.try_as_open()?; + + if let Some(connecting) = &state.connecting + && connecting.id == msg.id + { + state.connecting = None; + } + + Ok(()) + } +} + +impl Message for CameraFeed { + type Reply = (); + + async fn handle(&mut self, _: Unlock, _: &mut Context) -> Self::Reply { + trace!("CameraFeed.Unlock"); + + replace_with_or_abort(&mut self.state, |state| { + if let State::Locked { inner } = state { + State::Open(OpenState { + connecting: None, + attached: Some(inner), + }) + } else { + state + } + }); + } } diff --git a/crates/recording/src/feeds/mod.rs b/crates/recording/src/feeds/mod.rs index eacc27e846..3bf6a9a9cf 100644 --- a/crates/recording/src/feeds/mod.rs +++ b/crates/recording/src/feeds/mod.rs @@ -1,4 +1,2 @@ -mod camera; +pub mod camera; pub mod microphone; - -pub use camera::*; diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index 8ed6604fd0..a50a79df83 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -19,6 +19,8 @@ use sources::*; use std::sync::Arc; use thiserror::Error; +use crate::feeds::camera::CameraFeedLock; + #[derive(specta::Type, Serialize, Deserialize, Clone, Debug, Copy)] #[serde(rename_all = "camelCase")] pub enum RecordingMode { @@ -43,6 +45,7 @@ pub struct RecordingBaseInputs { pub capture_target: ScreenCaptureTarget, pub capture_system_audio: bool, pub mic_feed: Option>, + pub camera_feed: Option>, } #[derive(specta::Type, Serialize, Deserialize, Clone, Debug)] diff --git a/crates/recording/src/sources/audio_input.rs b/crates/recording/src/sources/audio_input.rs index 55771cbf7a..26adfacd6c 100644 --- a/crates/recording/src/sources/audio_input.rs +++ b/crates/recording/src/sources/audio_input.rs @@ -29,7 +29,7 @@ impl AudioInputSource { start_time: SystemTime, ) -> Self { Self { - audio_info: AudioInfo::from_stream_config(feed.config()), + audio_info: feed.audio_info().clone(), feed, tx, start_timestamp: None, diff --git a/crates/recording/src/sources/camera.rs b/crates/recording/src/sources/camera.rs index d1e49b32b2..3bbd406b15 100644 --- a/crates/recording/src/sources/camera.rs +++ b/crates/recording/src/sources/camera.rs @@ -1,17 +1,20 @@ use cap_media_info::VideoInfo; use ffmpeg::frame; use flume::{Receiver, Sender}; -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use tracing::{error, info}; use crate::{ MediaError, - feeds::{CameraConnection, CameraFeed, RawCameraFrame}, + feeds::camera::{self, CameraFeedLock, RawCameraFrame}, pipeline::{control::Control, task::PipelineSourceTask}, }; pub struct CameraSource { - feed_connection: CameraConnection, + feed: Arc, video_info: VideoInfo, output: Sender<(frame::Video, f64)>, first_frame_instant: Option, @@ -21,13 +24,13 @@ pub struct CameraSource { impl CameraSource { pub fn init( - feed: &CameraFeed, + feed: Arc, output: Sender<(frame::Video, f64)>, start_instant: Instant, ) -> Self { Self { - feed_connection: feed.create_connection(), - video_info: feed.video_info(), + video_info: feed.video_info().clone(), + feed, output, first_frame_instant: None, first_frame_timestamp: None, @@ -92,7 +95,6 @@ impl CameraSource { } impl PipelineSourceTask for CameraSource { - // #[tracing::instrument(skip_all)] fn run( &mut self, ready_signal: crate::pipeline::task::PipelineReadySignal, @@ -102,7 +104,11 @@ impl PipelineSourceTask for CameraSource { info!("Camera source ready"); - let frames = frames_rx.get_or_insert_with(|| self.feed_connection.attach()); + let frames = frames_rx.get_or_insert_with(|| { + let (tx, rx) = flume::bounded(5); + let _ = self.feed.ask(camera::AddSender(tx)).blocking_send(); + rx + }); ready_signal.send(Ok(())).unwrap(); diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 6226408721..006dda9eed 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1,8 +1,8 @@ use crate::{ ActorError, MediaError, RecordingBaseInputs, RecordingError, capture_pipeline::{MakeCapturePipeline, ScreenCaptureMethod, create_screen_capture}, - cursor::{CursorActor, Cursors /*spawn_cursor_recorder*/, spawn_cursor_recorder}, - feeds::{CameraFeed, microphone::MicrophoneFeedLock}, + cursor::{CursorActor, Cursors, spawn_cursor_recorder}, + feeds::{camera::CameraFeedLock, microphone::MicrophoneFeedLock}, pipeline::Pipeline, sources::{AudioInputSource, CameraSource, ScreenCaptureFormat, ScreenCaptureTarget}, }; @@ -18,7 +18,7 @@ use std::{ sync::Arc, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use tokio::sync::{Mutex, oneshot}; +use tokio::sync::oneshot; use tracing::{debug, info, trace}; #[allow(clippy::large_enum_variant)] @@ -130,7 +130,6 @@ pub async fn spawn_studio_recording_actor( id: String, recording_dir: PathBuf, base_inputs: RecordingBaseInputs, - camera_feed: Option>>, custom_cursor_capture: bool, ) -> Result<(StudioRecordingHandle, oneshot::Receiver>), SpawnStudioRecordingError> { @@ -149,8 +148,7 @@ pub async fn spawn_studio_recording_actor( let start_time = SystemTime::now(); let start_instant = Instant::now(); - if let Some(camera_feed) = &camera_feed { - let camera_feed = camera_feed.lock().await; + if let Some(camera_feed) = &base_inputs.camera_feed { debug!("camera device info: {:#?}", camera_feed.camera_info()); debug!("camera video info: {:#?}", camera_feed.video_info()); } @@ -162,10 +160,7 @@ pub async fn spawn_studio_recording_actor( let mut segment_pipeline_factory = SegmentPipelineFactory::new( segments_dir, cursors_dir, - base_inputs.capture_target.clone(), - base_inputs.mic_feed.clone(), - base_inputs.capture_system_audio, - camera_feed, + base_inputs.clone(), custom_cursor_capture, start_time, start_instant, @@ -566,10 +561,7 @@ async fn stop_recording( struct SegmentPipelineFactory { segments_dir: PathBuf, cursors_dir: PathBuf, - capture_target: ScreenCaptureTarget, - mic_feed: Option>, - capture_system_audio: bool, - camera_feed: Option>>, + base_inputs: RecordingBaseInputs, custom_cursor_capture: bool, start_time: SystemTime, start_instant: Instant, @@ -581,10 +573,7 @@ impl SegmentPipelineFactory { pub fn new( segments_dir: PathBuf, cursors_dir: PathBuf, - capture_target: ScreenCaptureTarget, - mic_feed: Option>, - capture_system_audio: bool, - camera_feed: Option>>, + base_inputs: RecordingBaseInputs, custom_cursor_capture: bool, start_time: SystemTime, start_instant: Instant, @@ -592,10 +581,7 @@ impl SegmentPipelineFactory { Self { segments_dir, cursors_dir, - capture_target, - mic_feed, - capture_system_audio, - camera_feed, + base_inputs, custom_cursor_capture, start_time, start_instant, @@ -618,10 +604,10 @@ impl SegmentPipelineFactory { &self.segments_dir, &self.cursors_dir, self.index, - self.capture_target.clone(), - self.mic_feed.clone(), - self.capture_system_audio, - self.camera_feed.as_deref(), + self.base_inputs.capture_target.clone(), + self.base_inputs.mic_feed.clone(), + self.base_inputs.capture_system_audio, + self.base_inputs.camera_feed.clone(), cursors, next_cursors_id, self.custom_cursor_capture, @@ -663,7 +649,7 @@ async fn create_segment_pipeline( capture_target: ScreenCaptureTarget, mic_feed: Option>, capture_system_audio: bool, - camera_feed: Option<&Mutex>, + camera_feed: Option>, prev_cursors: Cursors, next_cursors_id: u32, custom_cursor_capture: bool, @@ -699,12 +685,6 @@ async fn create_segment_pipeline( ) .await?; - let camera_feed = match camera_feed.as_ref() { - Some(camera_feed) => Some(camera_feed.lock().await), - None => None, - }; - let camera_feed = camera_feed.as_deref(); - let dir = ensure_dir(&segments_dir.join(format!("segment-{index}")))?; let mut pipeline_builder = Pipeline::builder(); From 3488de0e984f324e429f96a6773a2ca3eb243635 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 03:59:04 +0800 Subject: [PATCH 08/18] wip --- apps/desktop/src-tauri/src/lib.rs | 17 +---------------- crates/recording/src/feeds/camera.rs | 10 +--------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 2176ef92fb..b766917974 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1929,22 +1929,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); - let camera_feed = { - let (error_tx, error_rx) = flume::bounded(1); - - let mic_feed = CameraFeed::spawn(CameraFeed::new(error_tx)); - - // TODO: make this part of a global actor one day - tokio::spawn(async move { - let Ok(err) = error_rx.recv_async().await else { - return; - }; - - error!("Camera feed actor error: {err:?}"); - }); - - mic_feed - }; + let camera_feed = CameraFeed::spawn(CameraFeed::new()); let mic_feed = { let (error_tx, error_rx) = flume::bounded(1); diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 180386f64c..58177d3a52 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -15,8 +15,6 @@ use tracing::{debug, error, trace, warn}; use cap_camera_ffmpeg::*; -type StreamError = (); // TODO: Fix this - const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); #[derive(Clone)] @@ -30,7 +28,6 @@ pub struct RawCameraFrame { pub struct CameraFeed { state: State, senders: Vec>, - input_id_counter: u32, } enum State { @@ -82,14 +79,13 @@ struct AttachedState { } impl CameraFeed { - pub fn new(error_sender: flume::Sender) -> Self { + pub fn new() -> Self { Self { state: State::Open(OpenState { connecting: None, attached: None, }), senders: Vec::new(), - input_id_counter: 0, } } } @@ -253,7 +249,6 @@ async fn setup_camera( let Ok(mut ff_frame) = frame.to_ffmpeg() else { return; }; - dbg!(ff_frame.format()); ff_frame.set_pts(Some(frame.timestamp.as_micros() as i64)); @@ -304,9 +299,6 @@ impl Message for CameraFeed { let state = self.state.try_as_open()?; - // let id = self.input_id_counter; - // self.input_id_counter += 1; - let (internal_ready_tx, internal_ready_rx) = oneshot::channel::>(); let (ready_tx, ready_rx) = From 6cb36781dc21069a92897329c1f1672e250fac53 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 04:50:13 +0800 Subject: [PATCH 09/18] loading state + fix camera feed by cleanup up channel handling --- apps/desktop/src-tauri/src/camera.rs | 61 ++++++++++++--------------- apps/desktop/src-tauri/src/lib.rs | 24 ++++++----- apps/desktop/src-tauri/src/windows.rs | 8 +++- crates/recording/src/feeds/camera.rs | 33 +++++++++++++++ 4 files changed, 81 insertions(+), 45 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 540b9119d5..141df3d465 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -1,10 +1,14 @@ use anyhow::{Context, anyhow}; -use cap_recording::feeds::camera::RawCameraFrame; +use cap_recording::feeds::{ + self, + camera::{CameraFeed, RawCameraFrame}, +}; use ffmpeg::{ format::{self, Pixel}, frame, software::scaling, }; +use kameo::actor::ActorRef; use serde::{Deserialize, Serialize}; use specta::Type; use std::{sync::Arc, thread}; @@ -51,12 +55,6 @@ pub struct CameraPreviewState { pub struct CameraPreviewManager { store: Result>, String>, preview: Option, - // TODO: Reusing flume channels can be unsafe as the frames will only - // go to a single receiver, not all of them. - channel: ( - flume::Sender, - flume::Receiver, - ), } impl CameraPreviewManager { @@ -67,7 +65,6 @@ impl CameraPreviewManager { .build() .map_err(|err| format!("Error initializing camera preview store: {err}")), preview: None, - channel: flume::bounded(4), } } @@ -99,19 +96,18 @@ impl CameraPreviewManager { Ok(()) } - pub fn attach(&self) -> flume::Sender { - // Drain the channel so when the preview is opened it doesn't show an old frame. - while let Ok(_) = self.channel.1.try_recv() {} - - self.channel.0.clone() - } - pub fn is_initialized(&self) -> bool { self.preview.is_some() } /// Initialize the camera preview for a specific Tauri window - pub async fn init_window(&mut self, window: WebviewWindow) -> anyhow::Result<()> { + pub async fn init_window( + &mut self, + window: WebviewWindow, + actor: ActorRef, + ) -> anyhow::Result<()> { + let (camera_tx, camera_rx) = flume::bounded(4); + let default_state = self .get_state() .map_err(|err| error!("Error getting camera preview state: {err}")) @@ -122,7 +118,6 @@ impl CameraPreviewManager { InitializedCameraPreview::init_wgpu(window.clone(), default_state).await?; window.show().ok(); - let camera_rx = self.channel.1.clone(); let rt = Runtime::new().unwrap(); thread::spawn(move || { LocalSet::new().block_on(&rt, renderer.run(window, reconfigure_rx, camera_rx)) @@ -130,6 +125,11 @@ impl CameraPreviewManager { self.preview = Some(InitializedCameraPreview { reconfigure }); + actor + .ask(feeds::camera::AddSender(camera_tx)) + .await + .context("Error attaching camera feed consumer")?; + Ok(()) } @@ -156,9 +156,6 @@ impl CameraPreviewManager { .map_err(|err| error!("Error sending camera preview shutdown event: {err}")) .ok(); } - - // Drain the channel so when the preview is opened it doesn't show it. - while let Ok(_) = self.channel.1.try_recv() {} } } @@ -430,7 +427,7 @@ impl InitializedCameraPreview { .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) { let output_width = 5; - let output_height = (5.0 * 1.7777778) as u32; // TODO + let output_height = 5; let (buffer, stride) = render_solid_frame( [0x11, 0x11, 0x11, 0xFF], // #111111 @@ -450,8 +447,6 @@ impl InitializedCameraPreview { ) .render(&surface, &buffer, stride); surface.present(); - - println!("1: {:?} {:?} {:?}", size.0, size.1, aspect); } Ok(renderer) @@ -517,8 +512,6 @@ impl Renderer { let aspect_ratio = frame.frame.width() as f32 / frame.frame.height() as f32; self.sync_aspect_ratio_uniforms(aspect_ratio); - println!("2: {:?}", aspect_ratio); - if let Ok(surface) = self.surface.get_current_texture().map_err(|err| { error!("Error getting camera renderer surface texture: {err:?}") }) { @@ -569,15 +562,15 @@ impl Renderer { } Err(ReconfigureEvent::State(state)) => { // Aspect ratio is hardcoded until we can derive it from the camera feed - // if !seen_first_frame { // TODO - // self.sync_aspect_ratio_uniforms( - // if state.shape == CameraPreviewShape::Full { - // 16.0 / 9.0 - // } else { - // 1.0 - // }, - // ); - // } + if !seen_first_frame { + self.sync_aspect_ratio_uniforms( + if state.shape == CameraPreviewShape::Full { + 16.0 / 9.0 + } else { + 1.0 + }, + ); + } self.update_state_uniforms(&state); if let Some(aspect_ratio) = self.aspect_ratio.get_latest_key() { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6950a61e97..e70ae59c09 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -26,7 +26,6 @@ mod windows; use audio::AppSounds; use auth::{AuthStore, AuthenticationInvalid, Plan}; use camera::CameraPreviewState; -use cap_displays::{DisplayId, WindowId, bounds::LogicalBounds}; use cap_editor::{EditorInstance, EditorState}; use cap_project::{ ProjectConfiguration, RecordingMeta, RecordingMetaInner, SharingMeta, StudioRecordingMeta, XY, @@ -36,7 +35,7 @@ use cap_recording::{ RecordingMode, feeds::{ self, - camera::{CameraFeed, DeviceOrModelID, RawCameraFrame}, + camera::{CameraFeed, DeviceOrModelID}, microphone::{self, MicrophoneFeed}, }, sources::ScreenCaptureTarget, @@ -79,7 +78,7 @@ use tauri_plugin_notification::{NotificationExt, PermissionState}; use tauri_plugin_opener::OpenerExt; use tauri_plugin_shell::ShellExt; use tauri_specta::Event; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, oneshot}; use tracing::{error, trace}; use upload::{S3UploadMeta, create_or_get_video, upload_image, upload_video}; use web_api::ManagerExt as WebManagerExt; @@ -101,9 +100,6 @@ pub enum RecordingState { #[derive(specta::Type, Serialize)] #[serde(rename_all = "camelCase")] pub struct App { - #[serde(skip)] - #[deprecated = "can be removed when native camera preview is ready"] - camera_tx: flume::Sender, #[deprecated = "can be removed when native camera preview is ready"] camera_ws_port: u16, #[serde(skip)] @@ -248,7 +244,8 @@ async fn set_camera_input( state: MutableState<'_, App>, id: Option, ) -> Result<(), String> { - let camera_feed = state.read().await.camera_feed.clone(); + let app = state.read().await; + let camera_feed = app.camera_feed.clone(); match id { None => { @@ -1776,8 +1773,15 @@ async fn set_camera_preview_state( #[tauri::command] #[specta::specta] -async fn await_camera_preview_ready(app: MutableState<'_, App>) -> Result { - // store.wait_for_camera_to_load().await; // TODO: Reimplement this +async fn await_camera_preview_ready(app: MutableState<'_, App>) -> Result { + let app = app.read().await.camera_feed.clone(); + + let (tx, rx) = oneshot::channel(); + app.tell(feeds::camera::ListenForReady(tx)) + .await + .map_err(|err| format!("error registering ready listener: {err}"))?; + rx.await + .map_err(|err| format!("error receiving ready signal: {err}"))?; Ok(true) } @@ -1922,6 +1926,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); let camera_feed = CameraFeed::spawn(CameraFeed::new()); + let _ = camera_feed.ask(feeds::camera::AddSender(camera_tx)).await; let mic_feed = { let (error_tx, error_rx) = flume::bounded(1); @@ -2034,7 +2039,6 @@ pub async fn run(recording_logging_handle: LoggingHandle) { { app.manage(Arc::new(RwLock::new(App { - camera_tx, camera_ws_port, handle: app.clone(), camera_preview: CameraPreviewManager::new(&app), diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 42792e58df..3f77a95ac4 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -1,6 +1,7 @@ #![allow(unused_mut)] #![allow(unused_imports)] +use anyhow::anyhow; use futures::pin_mut; use scap_targets::{Display, DisplayId}; use serde::Deserialize; @@ -442,7 +443,12 @@ impl ShowCapWindow { let window = window_builder.build()?; if enable_native_camera_preview { - if let Err(err) = state.camera_preview.init_window(window.clone()).await { + let camera_feed = state.camera_feed.clone(); + if let Err(err) = state + .camera_preview + .init_window(window.clone(), camera_feed) + .await + { error!("Error initializing camera preview: {err}"); window.close().ok(); } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 58177d3a52..7a6b0f3f11 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -28,6 +28,7 @@ pub struct RawCameraFrame { pub struct CameraFeed { state: State, senders: Vec>, + on_ready: Vec>, } enum State { @@ -86,6 +87,7 @@ impl CameraFeed { attached: None, }), senders: Vec::new(), + on_ready: Vec::new(), } } } @@ -146,6 +148,8 @@ pub struct RemoveInput; pub struct AddSender(pub flume::Sender); +pub struct ListenForReady(pub oneshot::Sender<()>); + pub struct Lock; // Private Events @@ -373,10 +377,35 @@ impl Message for CameraFeed { } } +impl Message for CameraFeed { + type Reply = (); + + async fn handle( + &mut self, + msg: ListenForReady, + _: &mut Context, + ) -> Self::Reply { + match self.state { + State::Locked { .. } + | State::Open(OpenState { + connecting: None, + attached: Some(..), + }) => { + msg.0.send(()).ok(); + } + _ => { + self.on_ready.push(msg.0); + } + } + } +} + impl Message for CameraFeed { type Reply = (); async fn handle(&mut self, msg: NewFrame, _: &mut Context) -> Self::Reply { + println!("EMIT FRAME TO {}", self.senders.len()); + let mut to_remove = vec![]; for (i, sender) in self.senders.iter().enumerate() { @@ -462,6 +491,10 @@ impl Message for CameraFeed { } } + for tx in &mut self.on_ready.drain(..) { + tx.send(()).ok(); + } + Ok(()) } } From 16c1b01981ce784fe12f65def030758174834411 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 05:08:36 +0800 Subject: [PATCH 10/18] wip --- .../desktop/src-tauri/src/deeplink_actions.rs | 2 +- apps/desktop/src-tauri/src/lib.rs | 26 +++++++++++++++++++ crates/recording/src/feeds/camera.rs | 22 ++++++++++++++-- crates/recording/src/sources/camera.rs | 2 +- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src-tauri/src/deeplink_actions.rs b/apps/desktop/src-tauri/src/deeplink_actions.rs index a9e2b080dc..ff4f05a998 100644 --- a/apps/desktop/src-tauri/src/deeplink_actions.rs +++ b/apps/desktop/src-tauri/src/deeplink_actions.rs @@ -114,7 +114,7 @@ impl DeepLinkAction { } => { let state = app.state::>(); - crate::set_camera_input(state.clone(), camera).await?; + crate::set_camera_input(app.clone(), state.clone(), camera).await?; crate::set_mic_input(state.clone(), mic_label).await?; let capture_target: ScreenCaptureTarget = match capture_mode { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index e70ae59c09..df39988938 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -241,6 +241,7 @@ async fn set_mic_input(state: MutableState<'_, App>, label: Option) -> R #[tauri::command] #[specta::specta] async fn set_camera_input( + app_handle: AppHandle, state: MutableState<'_, App>, id: Option, ) -> Result<(), String> { @@ -255,6 +256,12 @@ async fn set_camera_input( .map_err(|e| e.to_string())?; } Some(id) => { + ShowCapWindow::Camera + .show(&app_handle) + .await + .map_err(|err| error!("Failed to show camera preview window: {err}")) + .ok(); + camera_feed .ask(feeds::camera::SetInput { id }) .await @@ -2028,6 +2035,25 @@ pub async fn run(recording_logging_handle: LoggingHandle) { app.manage(target_select_overlay::WindowFocusManager::default()); app.manage(EditorWindowIds::default()); + tokio::spawn({ + let camera_feed = camera_feed.clone(); + let app = app.clone(); + async move { + camera_feed + .tell(feeds::camera::OnFeedDisconnect(Box::new({ + move || { + if let Some(win) = CapWindowId::Camera.get(&app) { + win.close().ok(); + } + } + }))) + .send() + .await + .map_err(|err| error!("Error registering on camera feed disconnect: {err}")) + .ok(); + } + }); + if let Ok(Some(auth)) = AuthStore::load(&app) { sentry::configure_scope(|scope| { scope.set_user(auth.user_id.map(|id| sentry::User { diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 7a6b0f3f11..88d31b8d1e 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -29,6 +29,7 @@ pub struct CameraFeed { state: State, senders: Vec>, on_ready: Vec>, + on_disconnect: Vec>, } enum State { @@ -88,6 +89,7 @@ impl CameraFeed { }), senders: Vec::new(), on_ready: Vec::new(), + on_disconnect: Vec::new(), } } } @@ -150,6 +152,8 @@ pub struct AddSender(pub flume::Sender); pub struct ListenForReady(pub oneshot::Sender<()>); +pub struct OnFeedDisconnect(pub Box); + pub struct Lock; // Private Events @@ -365,6 +369,10 @@ impl Message for CameraFeed { let _ = handle.stop_capturing(); } + for cb in &self.on_disconnect { + (cb)(); + } + Ok(()) } } @@ -400,12 +408,22 @@ impl Message for CameraFeed { } } +impl Message for CameraFeed { + type Reply = (); + + async fn handle( + &mut self, + msg: OnFeedDisconnect, + _: &mut Context, + ) -> Self::Reply { + self.on_disconnect.push(msg.0); + } +} + impl Message for CameraFeed { type Reply = (); async fn handle(&mut self, msg: NewFrame, _: &mut Context) -> Self::Reply { - println!("EMIT FRAME TO {}", self.senders.len()); - let mut to_remove = vec![]; for (i, sender) in self.senders.iter().enumerate() { diff --git a/crates/recording/src/sources/camera.rs b/crates/recording/src/sources/camera.rs index 3bbd406b15..5d290bc1f4 100644 --- a/crates/recording/src/sources/camera.rs +++ b/crates/recording/src/sources/camera.rs @@ -29,7 +29,7 @@ impl CameraSource { start_instant: Instant, ) -> Self { Self { - video_info: feed.video_info().clone(), + video_info: *feed.video_info(), feed, output, first_frame_instant: None, From 323f220a587b40555c3a1dee8ab3e4e980698cc8 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 05:35:52 +0800 Subject: [PATCH 11/18] fix camera preview height strech --- apps/desktop/src-tauri/src/camera.rs | 98 ++++++++++++++-------------- apps/desktop/src-tauri/src/lib.rs | 14 ---- 2 files changed, 49 insertions(+), 63 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 141df3d465..f738b73e66 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -18,7 +18,7 @@ use tokio::{ sync::{broadcast, oneshot}, task::LocalSet, }; -use tracing::{error, info}; +use tracing::{error, info, trace}; use wgpu::{CompositeAlphaMode, SurfaceTexture}; static TOOLBAR_HEIGHT: f32 = 56.0; // also defined in Typescript @@ -115,12 +115,15 @@ impl CameraPreviewManager { let (reconfigure, reconfigure_rx) = broadcast::channel(1); let mut renderer = - InitializedCameraPreview::init_wgpu(window.clone(), default_state).await?; + InitializedCameraPreview::init_wgpu(window.clone(), &default_state).await?; window.show().ok(); - let rt = Runtime::new().unwrap(); + let rt = Runtime::new().expect("Failed to get Tokio runtime!"); thread::spawn(move || { - LocalSet::new().block_on(&rt, renderer.run(window, reconfigure_rx, camera_rx)) + LocalSet::new().block_on( + &rt, + renderer.run(window, default_state, reconfigure_rx, camera_rx), + ) }); self.preview = Some(InitializedCameraPreview { reconfigure }); @@ -133,19 +136,6 @@ impl CameraPreviewManager { Ok(()) } - /// Called by Tauri's event loop in response to a window resize event. - /// In theory if we get the event loop right this isn't required, - /// but it means if we mistake we get a small glitch instead of it - /// being permanently incorrectly sized or scaled. - pub fn on_window_resize(&self, width: u32, height: u32) { - if let Some(preview) = &self.preview { - preview - .reconfigure - .send(ReconfigureEvent::WindowSize(width, height)) - .ok(); - } - } - /// Called by Tauri's event loop in response to a window destroy event. pub fn on_window_close(&mut self) { if let Some(preview) = self.preview.take() { @@ -161,7 +151,6 @@ impl CameraPreviewManager { #[derive(Clone)] enum ReconfigureEvent { - WindowSize(u32, u32), State(CameraPreviewState), Shutdown, } @@ -173,7 +162,7 @@ struct InitializedCameraPreview { impl InitializedCameraPreview { async fn init_wgpu( window: WebviewWindow, - default_state: CameraPreviewState, + default_state: &CameraPreviewState, ) -> anyhow::Result { let aspect = if default_state.shape == CameraPreviewShape::Full { 16.0 / 9.0 @@ -416,7 +405,7 @@ impl InitializedCameraPreview { }; renderer.update_state_uniforms(&default_state); - renderer.sync_aspect_ratio_uniforms(aspect); + renderer.sync_ratio_uniform_and_resize_window_to_it(&window, &default_state, aspect); renderer.reconfigure_gpu_surface(size.0, size.1); // We initialize and render a blank color fallback. @@ -473,10 +462,10 @@ impl Renderer { async fn run( &mut self, window: WebviewWindow, + default_state: CameraPreviewState, mut reconfigure: broadcast::Receiver, camera_rx: flume::Receiver, ) { - // let mut window_size = (size.width, size.height); let mut resampler_frame = Cached::default(); let Ok(mut scaler) = scaling::Context::get( Pixel::RGBA, @@ -491,7 +480,7 @@ impl Renderer { return; }; - let mut seen_first_frame = false; + let mut state = default_state; while let Some(event) = loop { tokio::select! { frame = camera_rx.recv_async() => break frame.ok().map(Ok), @@ -506,11 +495,8 @@ impl Renderer { } { match event { Ok(frame) => { - if !seen_first_frame { - seen_first_frame = true; - } let aspect_ratio = frame.frame.width() as f32 / frame.frame.height() as f32; - self.sync_aspect_ratio_uniforms(aspect_ratio); + self.sync_ratio_uniform_and_resize_window_to_it(&window, &state, aspect_ratio); if let Ok(surface) = self.surface.get_current_texture().map_err(|err| { error!("Error getting camera renderer surface texture: {err:?}") @@ -557,28 +543,28 @@ impl Renderer { surface.present(); } } - Err(ReconfigureEvent::WindowSize(width, height)) => { - self.reconfigure_gpu_surface(width, height); - } - Err(ReconfigureEvent::State(state)) => { - // Aspect ratio is hardcoded until we can derive it from the camera feed - if !seen_first_frame { - self.sync_aspect_ratio_uniforms( - if state.shape == CameraPreviewShape::Full { - 16.0 / 9.0 - } else { - 1.0 - }, - ); - } - + Err(ReconfigureEvent::State(new_state)) => { + trace!("CameraPreview/ReconfigureEvent.State({new_state:?})"); + + state = new_state; + + let aspect_ratio = self + .aspect_ratio + .get_latest_key() + .copied() + // Aspect ratio is hardcoded until we can derive it from the camera feed + .unwrap_or(if state.shape == CameraPreviewShape::Full { + 16.0 / 9.0 + } else { + 1.0 + }); + + self.sync_ratio_uniform_and_resize_window_to_it(&window, &state, aspect_ratio); self.update_state_uniforms(&state); - if let Some(aspect_ratio) = self.aspect_ratio.get_latest_key() { - if let Ok((width, height)) = resize_window(&window, &state, *aspect_ratio) - .map_err(|err| error!("Error resizing camera preview window: {err}")) - { - self.reconfigure_gpu_surface(width, height); - } + if let Ok((width, height)) = resize_window(&window, &state, aspect_ratio) + .map_err(|err| error!("Error resizing camera preview window: {err}")) + { + self.reconfigure_gpu_surface(width, height); } } Err(ReconfigureEvent::Shutdown) => return, @@ -640,8 +626,14 @@ impl Renderer { ); } - /// Update the uniforms which hold the camera aspect ratio if it's changed - fn sync_aspect_ratio_uniforms(&mut self, aspect_ratio: f32) { + /// Update the uniforms which hold the camera aspect ratio if it's changed, + /// and resize the window to match the new aspect ratio if required. + fn sync_ratio_uniform_and_resize_window_to_it( + &mut self, + window: &WebviewWindow, + state: &CameraPreviewState, + aspect_ratio: f32, + ) { if self.aspect_ratio.update_key_and_should_init(aspect_ratio) { let camera_uniforms = CameraUniforms { camera_aspect_ratio: aspect_ratio, @@ -652,6 +644,12 @@ impl Renderer { 0, bytemuck::cast_slice(&[camera_uniforms]), ); + + if let Ok((width, height)) = resize_window(&window, &state, aspect_ratio) + .map_err(|err| error!("Error resizing camera preview window: {err}")) + { + self.reconfigure_gpu_surface(width, height); + } } } } @@ -663,6 +661,8 @@ fn resize_window( state: &CameraPreviewState, aspect: f32, ) -> tauri::Result<(u32, u32)> { + trace!("CameraPreview/resize_window"); + let base: f32 = if state.size == CameraPreviewSize::Sm { 230.0 } else { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index df39988938..4bd95860ac 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2268,20 +2268,6 @@ pub async fn run(recording_logging_handle: LoggingHandle) { api.prevent_exit(); } } - tauri::RunEvent::WindowEvent { - event: WindowEvent::Resized(size), - .. - } => { - let handle = handle.clone(); - tokio::spawn(async move { - handle - .state::>() - .read() - .await - .camera_preview - .on_window_resize(size.width, size.height) - }); - } _ => {} }); } From 98390793a49b1beeff4d6329bc4f4c4f1138b367 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 05:38:31 +0800 Subject: [PATCH 12/18] format --- apps/desktop/src/utils/tauri.ts | 8 ++++---- crates/recording/src/feeds/camera.rs | 5 ++--- crates/recording/src/sources/camera.rs | 4 ++-- crates/rendering/src/layout.rs | 6 +----- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 1c0aaf2eea..5c25224f53 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -6,7 +6,7 @@ export const commands = { async setMicInput(label: string | null): Promise { return await TAURI_INVOKE("set_mic_input", { label }); }, - async setCameraInput(id: DeviceOrModelID | null): Promise { + async setCameraInput(id: DeviceOrModelID | null): Promise { return await TAURI_INVOKE("set_camera_input", { id }); }, async startRecording(inputs: StartRecordingInputs): Promise { @@ -222,7 +222,7 @@ export const commands = { async setServerUrl(serverUrl: string): Promise { return await TAURI_INVOKE("set_server_url", { serverUrl }); }, - async setCameraPreviewState(state: CameraWindowState): Promise { + async setCameraPreviewState(state: CameraPreviewState): Promise { return await TAURI_INVOKE("set_camera_preview_state", { state }); }, async awaitCameraPreviewReady(): Promise { @@ -432,12 +432,12 @@ export type CameraInfo = { export type CameraPosition = { x: CameraXPosition; y: CameraYPosition }; export type CameraPreviewShape = "round" | "square" | "full"; export type CameraPreviewSize = "sm" | "lg"; -export type CameraShape = "square" | "source"; -export type CameraWindowState = { +export type CameraPreviewState = { size: CameraPreviewSize; shape: CameraPreviewShape; mirrored: boolean; }; +export type CameraShape = "square" | "source"; export type CameraXPosition = "left" | "center" | "right"; export type CameraYPosition = "top" | "bottom"; export type CaptionData = { diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 88d31b8d1e..7be9d1face 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -21,7 +21,7 @@ const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); pub struct RawCameraFrame { pub frame: frame::Video, pub timestamp: Duration, - pub refrence_time: Instant, + pub reference_time: Instant, } #[derive(Actor)] @@ -275,7 +275,7 @@ async fn setup_camera( .tell(NewFrame(RawCameraFrame { frame: ff_frame, timestamp: frame.timestamp, - refrence_time: frame.reference_time, + reference_time: frame.reference_time, })) .try_send(); }) @@ -504,7 +504,6 @@ impl Message for CameraFeed { let res = ready.await; if let Ok(data) = res { - println!("connected: {:?}", &id); state.handle_input_connected(data, id); } } diff --git a/crates/recording/src/sources/camera.rs b/crates/recording/src/sources/camera.rs index 5d290bc1f4..0580428527 100644 --- a/crates/recording/src/sources/camera.rs +++ b/crates/recording/src/sources/camera.rs @@ -81,7 +81,7 @@ impl CameraSource { drop(frames_rx); for frame in frames { - let first_frame_instant = *self.first_frame_instant.get_or_insert(frame.refrence_time); + let first_frame_instant = *self.first_frame_instant.get_or_insert(frame.reference_time); let first_frame_timestamp = *self.first_frame_timestamp.get_or_insert(frame.timestamp); if let Err(error) = @@ -117,7 +117,7 @@ impl PipelineSourceTask for CameraSource { Some(Control::Play) => match frames.drain().last().or_else(|| frames.recv().ok()) { Some(frame) => { let first_frame_instant = - *self.first_frame_instant.get_or_insert(frame.refrence_time); + *self.first_frame_instant.get_or_insert(frame.reference_time); let first_frame_timestamp = *self.first_frame_timestamp.get_or_insert(frame.timestamp); diff --git a/crates/rendering/src/layout.rs b/crates/rendering/src/layout.rs index 0e0eb5745b..1e6619dcaa 100644 --- a/crates/rendering/src/layout.rs +++ b/crates/rendering/src/layout.rs @@ -76,11 +76,7 @@ impl InterpolatedLayout { .map(|s| s.mode) .unwrap_or(LayoutMode::Default); let progress = (cursor.time - transition_start) / LAYOUT_TRANSITION_DURATION; - ( - prev_mode, - segment.mode, - ease_in_out(progress as f32) as f64, - ) + (prev_mode, segment.mode, ease_in_out(progress as f32) as f64) } else if cursor.time >= transition_end && cursor.time < segment.end { if let Some(next_seg) = cursor.next_segment() { let progress = (cursor.time - transition_end) / LAYOUT_TRANSITION_DURATION; From 358fb420930aacfb14eade7cff49f5461ad47f77 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 05:50:29 +0800 Subject: [PATCH 13/18] Clippy --- apps/cli/src/main.rs | 3 +-- apps/cli/src/record.rs | 25 +++++++++---------- apps/desktop/src-tauri/src/camera.rs | 11 ++++---- apps/desktop/src-tauri/src/lib.rs | 8 +++--- apps/desktop/src-tauri/src/windows.rs | 4 +-- crates/recording/src/feeds/camera.rs | 9 ++++--- crates/recording/src/pipeline/builder.rs | 8 +----- crates/recording/src/pipeline/mod.rs | 2 +- crates/recording/src/sources/audio_input.rs | 2 +- .../src/sources/screen_capture/macos.rs | 7 +++--- .../src/sources/screen_capture/mod.rs | 8 +++++- 11 files changed, 44 insertions(+), 43 deletions(-) diff --git a/apps/cli/src/main.rs b/apps/cli/src/main.rs index 00f7a959c8..b82ac725fe 100644 --- a/apps/cli/src/main.rs +++ b/apps/cli/src/main.rs @@ -7,7 +7,6 @@ use std::{ use cap_export::ExporterBase; use cap_project::XY; -use cap_recording::feeds::CameraFeed; use clap::{Args, Parser, Subcommand}; use record::RecordStart; use serde_json::json; @@ -111,7 +110,7 @@ window {}: } } Some(RecordCommands::Cameras) => { - let cameras = CameraFeed::list_cameras(); + let cameras = cap_camera::list_cameras().collect::>(); let mut info = vec![]; for camera_info in cameras { diff --git a/apps/cli/src/record.rs b/apps/cli/src/record.rs index 5aa545a64f..f05d588869 100644 --- a/apps/cli/src/record.rs +++ b/apps/cli/src/record.rs @@ -1,9 +1,8 @@ -use cap_camera::ModelID; use cap_recording::screen_capture::ScreenCaptureTarget; use clap::Args; use scap_targets::{DisplayId, WindowId}; -use std::{env::current_dir, path::PathBuf, sync::Arc}; -use tokio::{io::AsyncBufReadExt, sync::Mutex}; +use std::{env::current_dir, path::PathBuf}; +use tokio::io::AsyncBufReadExt; use uuid::Uuid; #[derive(Args)] @@ -43,16 +42,16 @@ impl RecordStart { _ => Err("No target specified".to_string()), }?; - let camera = if let Some(model_id) = self.camera { - let _model_id: ModelID = model_id - .try_into() - .map_err(|_| "Invalid model ID".to_string())?; + // let camera = if let Some(model_id) = self.camera { + // let _model_id: ModelID = model_id + // .try_into() + // .map_err(|_| "Invalid model ID".to_string())?; - todo!() - // Some(CameraFeed::init(model_id).await.unwrap()) - } else { - None - }; + // todo!() + // // Some(CameraFeed::init(model_id).await.unwrap()) + // } else { + // None + // }; let id = Uuid::new_v4().to_string(); let path = self @@ -66,8 +65,8 @@ impl RecordStart { capture_target: target_info, capture_system_audio: self.system_audio, mic_feed: None, + camera_feed: None, // camera.map(|c| Arc::new(Mutex::new(c))), }, - camera.map(|c| Arc::new(Mutex::new(c))), false, ) .await diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index f738b73e66..5b6e6546a8 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -170,8 +170,8 @@ impl InitializedCameraPreview { 1.0 }; - let size = resize_window(&window, &default_state, aspect) - .context("Error resizing Tauri window")?; + let size = + resize_window(&window, default_state, aspect).context("Error resizing Tauri window")?; let (tx, rx) = oneshot::channel(); window @@ -404,8 +404,8 @@ impl InitializedCameraPreview { aspect_ratio: Cached::default(), }; - renderer.update_state_uniforms(&default_state); - renderer.sync_ratio_uniform_and_resize_window_to_it(&window, &default_state, aspect); + renderer.update_state_uniforms(default_state); + renderer.sync_ratio_uniform_and_resize_window_to_it(&window, default_state, aspect); renderer.reconfigure_gpu_surface(size.0, size.1); // We initialize and render a blank color fallback. @@ -645,7 +645,7 @@ impl Renderer { bytemuck::cast_slice(&[camera_uniforms]), ); - if let Ok((width, height)) = resize_window(&window, &state, aspect_ratio) + if let Ok((width, height)) = resize_window(window, state, aspect_ratio) .map_err(|err| error!("Error resizing camera preview window: {err}")) { self.reconfigure_gpu_surface(width, height); @@ -726,6 +726,7 @@ pub struct PreparedTexture { } impl PreparedTexture { + #[allow(clippy::too_many_arguments)] pub fn init( device: wgpu::Device, queue: wgpu::Queue, diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 4bd95860ac..896e9a0f3a 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -386,14 +386,14 @@ async fn get_current_recording( ScreenCaptureTarget::Display { id } => CurrentRecordingTarget::Screen { id: id.clone() }, ScreenCaptureTarget::Window { id } => CurrentRecordingTarget::Window { id: id.clone(), - bounds: scap_targets::Window::from_id(&id) + bounds: scap_targets::Window::from_id(id) .ok_or(())? .display_relative_logical_bounds() .ok_or(())?, }, ScreenCaptureTarget::Area { screen, bounds } => CurrentRecordingTarget::Area { screen: screen.clone(), - bounds: bounds.clone(), + bounds: *bounds, }, }; @@ -1058,7 +1058,7 @@ async fn upload_exported_video( } let metadata = build_video_meta(&output_path) - .map_err(|err| format!("Error getting output video meta: {}", err.to_string()))?; + .map_err(|err| format!("Error getting output video meta: {err}"))?; if !auth.is_upgraded() && metadata.duration_in_secs > 300.0 { return Ok(UploadResult::UpgradeRequired); @@ -1932,7 +1932,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); - let camera_feed = CameraFeed::spawn(CameraFeed::new()); + let camera_feed = CameraFeed::spawn(CameraFeed::default()); let _ = camera_feed.ask(feeds::camera::AddSender(camera_tx)).await; let mic_feed = { diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 3f77a95ac4..9cabf83137 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -395,7 +395,7 @@ impl ShowCapWindow { Self::Camera => { const WINDOW_SIZE: f64 = 230.0 * 2.0; - let enable_native_camera_preview = GeneralSettingsStore::get(&app) + let enable_native_camera_preview = GeneralSettingsStore::get(app) .ok() .and_then(|v| v.map(|v| v.enable_native_camera_preview)) .unwrap_or_default(); @@ -406,7 +406,7 @@ impl ShowCapWindow { if enable_native_camera_preview && state.camera_preview.is_initialized() { error!("Unable to initialize camera preview as one already exists!"); - if let Some(window) = CapWindowId::Camera.get(&app) { + if let Some(window) = CapWindowId::Camera.get(app) { window.show().ok(); } return Err(anyhow!( diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 7be9d1face..5f536b052b 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -74,14 +74,15 @@ struct ConnectingState { } struct AttachedState { + #[allow(dead_code)] id: DeviceOrModelID, handle: cap_camera::CaptureHandle, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, } -impl CameraFeed { - pub fn new() -> Self { +impl Default for CameraFeed { + fn default() -> Self { Self { state: State::Open(OpenState { connecting: None, @@ -337,7 +338,7 @@ impl Message for CameraFeed { tokio::spawn(async move { match setup_camera(&id, new_frame_recipient).await { Ok(r) => { - let _ = ready_tx.send(Ok((r.camera_info.clone(), r.video_info.clone()))); + let _ = ready_tx.send(Ok((r.camera_info.clone(), r.video_info))); let _ = internal_ready_tx.send(Ok(r)); let _ = actor_ref.ask(InputConnected).await; @@ -473,7 +474,7 @@ impl Message for CameraFeed { }; let camera_info = attached.camera_info.clone(); - let video_info = attached.video_info.clone(); + let video_info = attached.video_info; self.state = State::Locked { inner: attached }; diff --git a/crates/recording/src/pipeline/builder.rs b/crates/recording/src/pipeline/builder.rs index 6bcba23daf..a1b8f53193 100644 --- a/crates/recording/src/pipeline/builder.rs +++ b/crates/recording/src/pipeline/builder.rs @@ -19,19 +19,13 @@ struct Task { done_rx: tokio::sync::oneshot::Receiver>, } +#[derive(Default)] pub struct PipelineBuilder { control: ControlBroadcast, tasks: IndexMap, } impl PipelineBuilder { - pub fn new() -> Self { - Self { - control: ControlBroadcast::default(), - tasks: IndexMap::new(), - } - } - pub fn spawn_source( &mut self, name: impl Into, diff --git a/crates/recording/src/pipeline/mod.rs b/crates/recording/src/pipeline/mod.rs index 4ba97466e0..4d2e16c235 100644 --- a/crates/recording/src/pipeline/mod.rs +++ b/crates/recording/src/pipeline/mod.rs @@ -20,7 +20,7 @@ pub struct Pipeline { impl Pipeline { pub fn builder() -> PipelineBuilder { - PipelineBuilder::new() + PipelineBuilder::default() } pub async fn play(&mut self) -> Result<(), MediaError> { diff --git a/crates/recording/src/sources/audio_input.rs b/crates/recording/src/sources/audio_input.rs index 26adfacd6c..67416adc98 100644 --- a/crates/recording/src/sources/audio_input.rs +++ b/crates/recording/src/sources/audio_input.rs @@ -29,7 +29,7 @@ impl AudioInputSource { start_time: SystemTime, ) -> Self { Self { - audio_info: feed.audio_info().clone(), + audio_info: *feed.audio_info(), feed, tx, start_timestamp: None, diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index 4cfdf16b3f..c9838f9706 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -214,7 +214,7 @@ impl PipelineSourceTask for ScreenCaptureSource { .map_err(SourceError::CreateActor)?, ); - let _ = capturer + capturer .ask(StartCapturing) .await .map_err(SourceError::StartCapturing)?; @@ -282,10 +282,11 @@ impl ScreenCaptureActor { let capturer_builder = scap_screencapturekit::Capturer::builder(target, settings) .with_output_sample_buf_cb(move |frame| { let check_err = || { - Result::<_, arc::R>::Ok(cap_fail::fail_err!( + cap_fail::fail_err!( "macos::ScreenCaptureActor output_sample_buf", ns::Error::with_domain(ns::ErrorDomain::os_status(), 69420, None) - )) + ); + Result::<_, arc::R>::Ok(()) }; if let Err(e) = check_err() { let _ = _error_tx.send(e); diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index bf6ee04120..56794362c8 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -77,6 +77,7 @@ impl ScreenCaptureTarget { match self { Self::Display { .. } => { #[cfg(target_os = "macos")] + #[allow(clippy::needless_return)] { let display = self.display()?; return Some(CursorCropBounds::new_macos(LogicalBounds::new( @@ -86,6 +87,7 @@ impl ScreenCaptureTarget { } #[cfg(windows)] + #[allow(clippy::needless_return)] { let display = self.display()?; return Some(CursorCropBounds::new_windows(PhysicalBounds::new( @@ -98,6 +100,7 @@ impl ScreenCaptureTarget { let window = Window::from_id(id)?; #[cfg(target_os = "macos")] + #[allow(clippy::needless_return)] { let display = self.display()?; let display_position = display.raw_handle().logical_position(); @@ -113,6 +116,7 @@ impl ScreenCaptureTarget { } #[cfg(windows)] + #[allow(clippy::needless_return)] { let display_bounds = self.display()?.raw_handle().physical_bounds()?; let window_bounds = window.raw_handle().physical_bounds()?; @@ -131,11 +135,13 @@ impl ScreenCaptureTarget { } Self::Area { bounds, .. } => { #[cfg(target_os = "macos")] + #[allow(clippy::needless_return)] { return Some(CursorCropBounds::new_macos(*bounds)); } #[cfg(windows)] + #[allow(clippy::needless_return)] { let display = self.display()?; let display_bounds = display.raw_handle().physical_bounds()?; @@ -275,7 +281,7 @@ impl ScreenCaptureSource { let crop_bounds = match target { ScreenCaptureTarget::Display { .. } => None, ScreenCaptureTarget::Window { id } => { - let window = Window::from_id(&id).ok_or(ScreenCaptureInitError::NoWindow)?; + let window = Window::from_id(id).ok_or(ScreenCaptureInitError::NoWindow)?; #[cfg(target_os = "macos")] { From 0042257845626dd0ba02cf59ed48ed629a231404 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 11:24:15 +0800 Subject: [PATCH 14/18] fix --- apps/desktop/src-tauri/src/lib.rs | 1 + apps/desktop/src-tauri/src/windows.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 896e9a0f3a..e7a69027fa 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2178,6 +2178,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { id, CapWindowId::TargetSelectOverlay { .. } | CapWindowId::Main + | CapWindowId::Camera ) { let _ = window.show(); diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 9cabf83137..dc5bda851d 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -329,7 +329,9 @@ impl ShowCapWindow { if let Ok(id) = CapWindowId::from_str(&label) && matches!( id, - CapWindowId::TargetSelectOverlay { .. } | CapWindowId::Main + CapWindowId::TargetSelectOverlay { .. } + | CapWindowId::Main + | CapWindowId::Camera ) { let _ = window.hide(); From 58ebbfe465dd22097d6bdf468fa66e951f77712f Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Tue, 26 Aug 2025 13:09:54 +0800 Subject: [PATCH 15/18] allow disabling countdown --- apps/desktop/src/routes/target-select-overlay.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/desktop/src/routes/target-select-overlay.tsx b/apps/desktop/src/routes/target-select-overlay.tsx index 9c841f32e8..c49b9d14c9 100644 --- a/apps/desktop/src/routes/target-select-overlay.tsx +++ b/apps/desktop/src/routes/target-select-overlay.tsx @@ -691,6 +691,13 @@ function RecordingControls(props: { target: ScreenCaptureTarget }) { await Submenu.new({ text: "Recording Countdown", items: [ + await CheckMenuItem.new({ + text: "Off", + action: () => generalSettingsStore.set({ recordingCountdown: 0 }), + checked: + !generalSetings.data?.recordingCountdown || + generalSetings.data?.recordingCountdown === 0, + }), await CheckMenuItem.new({ text: "3 seconds", action: () => generalSettingsStore.set({ recordingCountdown: 3 }), From 49f02952bbf829e8a9ba6a3797ce7f1933a3e139 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 26 Aug 2025 13:26:25 +0800 Subject: [PATCH 16/18] unlock feeds via channel instead of blocking --- crates/recording/src/feeds/camera.rs | 16 +++++++++++++--- crates/recording/src/feeds/microphone.rs | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 5f536b052b..f7b750dd59 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -100,7 +100,7 @@ pub struct CameraFeedLock { actor: ActorRef, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, - lock_tx: Recipient, + drop_tx: Option>, } impl CameraFeedLock { @@ -123,7 +123,9 @@ impl Deref for CameraFeedLock { impl Drop for CameraFeedLock { fn drop(&mut self) { - let _ = self.lock_tx.tell(Unlock).blocking_send(); + if let Some(drop_tx) = self.drop_tx.take() { + let _ = drop_tx.send(()); + } } } @@ -478,11 +480,19 @@ impl Message for CameraFeed { self.state = State::Locked { inner: attached }; + let (drop_tx, drop_rx) = oneshot::channel(); + + let actor_ref = ctx.actor_ref(); + tokio::spawn(async move { + let _ = drop_rx.await; + let _ = actor_ref.tell(Unlock).await; + }); + Ok(CameraFeedLock { camera_info, video_info, actor: ctx.actor_ref(), - lock_tx: ctx.actor_ref().recipient(), + drop_tx: Some(drop_tx), }) } } diff --git a/crates/recording/src/feeds/microphone.rs b/crates/recording/src/feeds/microphone.rs index 75ba267327..d20b90092b 100644 --- a/crates/recording/src/feeds/microphone.rs +++ b/crates/recording/src/feeds/microphone.rs @@ -157,7 +157,7 @@ pub struct MicrophoneFeedLock { actor: ActorRef, config: SupportedStreamConfig, audio_info: AudioInfo, - lock_tx: Recipient, + drop_tx: Option>, } impl MicrophoneFeedLock { @@ -180,7 +180,9 @@ impl Deref for MicrophoneFeedLock { impl Drop for MicrophoneFeedLock { fn drop(&mut self) { - let _ = self.lock_tx.tell(Unlock).blocking_send(); + if let Some(drop_tx) = self.drop_tx.take() { + let _ = drop_tx.send(()); + } } } @@ -442,11 +444,19 @@ impl Message for MicrophoneFeed { self.state = State::Locked { inner: attached }; + let (drop_tx, drop_rx) = oneshot::channel(); + + let actor_ref = ctx.actor_ref(); + tokio::spawn(async move { + let _ = drop_rx.await; + let _ = actor_ref.tell(Unlock).await; + }); + Ok(MicrophoneFeedLock { audio_info: AudioInfo::from_stream_config(&config), actor: ctx.actor_ref(), config, - lock_tx: ctx.actor_ref().recipient(), + drop_tx: Some(drop_tx), }) } } From 9750509aca73cf0b1e532e5fa3f61cf98ba9c9bb Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 26 Aug 2025 14:08:23 +0800 Subject: [PATCH 17/18] fix camera feed on windows --- .../routes/(window-chrome)/new-main/index.tsx | 11 +- crates/recording/src/feeds/camera.rs | 102 ++++++++++-------- 2 files changed, 65 insertions(+), 48 deletions(-) diff --git a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx index f16a7f4f8d..fb47c8e198 100644 --- a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx +++ b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx @@ -292,11 +292,7 @@ function Page() { return (
-
+
Settings}>
-
+ {ostype() === "macos" && ( +
+ )} }> {license.data?.type === "commercial" diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index f7b750dd59..16660407f6 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -8,9 +8,10 @@ use replace_with::replace_with_or_abort; use std::{ cmp::Ordering, ops::Deref, + sync::mpsc::{self, SyncSender}, time::{Duration, Instant}, }; -use tokio::sync::oneshot; +use tokio::{runtime::Runtime, sync::oneshot, task::LocalSet}; use tracing::{debug, error, trace, warn}; use cap_camera_ffmpeg::*; @@ -53,15 +54,15 @@ struct OpenState { } impl OpenState { - fn handle_input_connected(&mut self, data: SetupCameraResult, id: DeviceOrModelID) { + fn handle_input_connected(&mut self, data: InputConnected, id: DeviceOrModelID) { if let Some(connecting) = &self.connecting && id == connecting.id { self.attached = Some(AttachedState { id, - handle: data.handle, camera_info: data.camera_info, video_info: data.video_info, + done_tx: data.done_tx, }); self.connecting = None; } @@ -70,15 +71,15 @@ impl OpenState { struct ConnectingState { id: DeviceOrModelID, - ready: BoxFuture<'static, Result>, + ready: BoxFuture<'static, Result>, } struct AttachedState { #[allow(dead_code)] id: DeviceOrModelID, - handle: cap_camera::CaptureHandle, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, + done_tx: mpsc::SyncSender<()>, } impl Default for CameraFeed { @@ -162,7 +163,11 @@ pub struct Lock; // Private Events #[derive(Clone)] -struct InputConnected; +struct InputConnected { + done_tx: SyncSender<()>, + camera_info: cap_camera::CameraInfo, + video_info: VideoInfo, +} struct InputConnectFailed { id: DeviceOrModelID, @@ -310,51 +315,64 @@ impl Message for CameraFeed { let state = self.state.try_as_open()?; - let (internal_ready_tx, internal_ready_rx) = - oneshot::channel::>(); - let (ready_tx, ready_rx) = - oneshot::channel::>(); + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(0); - let ready = { - ready_rx - .map(|v| { - v.map_err(|_| SetInputError::BuildStreamCrashed) - .and_then(|v| v) - }) - .shared() - }; + let ready = ready_rx + .map(|v| { + v.map_err(|_| SetInputError::BuildStreamCrashed) + .and_then(|v| v) + }) + .shared(); state.connecting = Some(ConnectingState { id: msg.id.clone(), - ready: internal_ready_rx - .map(|v| { - v.map_err(|_| SetInputError::BuildStreamCrashed) - .and_then(|v| v) - }) - .boxed(), + ready: ready.clone().boxed(), }); let id = msg.id.clone(); let actor_ref = ctx.actor_ref(); let new_frame_recipient = actor_ref.clone().recipient(); - tokio::spawn(async move { - match setup_camera(&id, new_frame_recipient).await { - Ok(r) => { - let _ = ready_tx.send(Ok((r.camera_info.clone(), r.video_info))); - let _ = internal_ready_tx.send(Ok(r)); - - let _ = actor_ref.ask(InputConnected).await; - } - Err(e) => { - let _ = ready_tx.send(Err(e.clone())); - let _ = internal_ready_tx.send(Err(e)); - - let _ = actor_ref.tell(InputConnectFailed { id }).await; - } - } + + let rt = Runtime::new().expect("Failed to get Tokio runtime!"); + std::thread::spawn(move || { + LocalSet::new().block_on(&rt, async move { + let handle = match setup_camera(&id, new_frame_recipient).await { + Ok(r) => { + let _ = ready_tx.send(Ok(InputConnected { + camera_info: r.camera_info.clone(), + video_info: r.video_info.clone(), + done_tx: done_tx.clone(), + })); + + let _ = actor_ref + .ask(InputConnected { + camera_info: r.camera_info.clone(), + video_info: r.video_info.clone(), + done_tx: done_tx.clone(), + }) + .await; + + r.handle + } + Err(e) => { + let _ = ready_tx.send(Err(e.clone())); + + let _ = actor_ref.tell(InputConnectFailed { id }).await; + + return; + } + }; + + let _ = done_rx.recv(); + + let _ = handle.stop_capturing(); + }) }); - Ok(ready.map(|v| v).boxed()) + Ok(ready + .map(|v| v.map(|v| (v.camera_info, v.video_info))) + .boxed()) } } @@ -368,8 +386,8 @@ impl Message for CameraFeed { state.connecting = None; - if let Some(AttachedState { handle, .. }) = state.attached.take() { - let _ = handle.stop_capturing(); + if let Some(AttachedState { done_tx, .. }) = state.attached.take() { + let _ = done_tx.send(()); } for cb in &self.on_disconnect { From d2b35fddf1a8c3e6e8fed09178bf03163ac9e377 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 26 Aug 2025 14:13:42 +0800 Subject: [PATCH 18/18] reverse header on macos --- .../desktop/src/routes/(window-chrome)/new-main/index.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx index fb47c8e198..9f9293a5be 100644 --- a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx +++ b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx @@ -292,7 +292,13 @@ function Page() { return (
-
+
Settings}>