Add output size support to H264 encoder

Author: richiemcilroy

crates/enc-ffmpeg/src/video/h264.rs

@@ -16,6 +16,7 @@ pub struct H264EncoderBuilder {
     bpp: f32,
     input_config: VideoInfo,
     preset: H264Preset,
+    output_size: Option<(u32, u32)>,
 }
 
 #[derive(Clone, Copy)]
@@ -43,6 +44,7 @@ impl H264EncoderBuilder {
             input_config,
             bpp: Self::QUALITY_BPP,
             preset: H264Preset::Ultrafast,
+            output_size: None,
         }
     }
 
@@ -56,45 +58,95 @@ impl H264EncoderBuilder {
         self
     }
 
+    pub fn with_output_size(mut self, width: u32, height: u32) -> Self {
+        if width == 0 || height == 0 {
+            self.output_size = None;
+        } else {
+            self.output_size = Some((width, height));
+        }
+        self
+    }
+
     pub fn build(
         self,
         output: &mut format::context::Output,
     ) -> Result<H264Encoder, H264EncoderError> {
-        let input_config = &self.input_config;
-        let (codec, encoder_options) = get_codec_and_options(input_config, self.preset)
+        let input_config = self.input_config;
+        let (codec, encoder_options) = get_codec_and_options(&input_config, self.preset)
             .ok_or(H264EncoderError::CodecNotFound)?;
 
-        let (format, converter) = if !codec
-            .video()
-            .unwrap()
+        let (mut output_width, mut output_height) = self
+            .output_size
+            .unwrap_or((input_config.width, input_config.height));
+
+        if output_width == 0 || output_height == 0 {
+            output_width = input_config.width;
+            output_height = input_config.height;
+        }
+
+        let codec_video = codec.video().unwrap();
+        let encoder_supports_input_format = codec_video
             .formats()
             .unwrap()
-            .any(|f| f == input_config.pixel_format)
-        {
+            .any(|f| f == input_config.pixel_format);
+
+        let mut needs_pixel_conversion = false;
+
+        let output_format = if encoder_supports_input_format {
+            input_config.pixel_format
+        } else {
+            needs_pixel_conversion = true;
             let format = ffmpeg::format::Pixel::NV12;
             debug!(
                 "Converting from {:?} to {:?} for H264 encoding",
                 input_config.pixel_format, format
             );
-            (
-                format,
-                Some(
-                    ffmpeg::software::converter(
-                        (input_config.width, input_config.height),
-                        input_config.pixel_format,
-                        format,
-                    )
-                    .map_err(|e| {
+            format
+        };
+
+        let needs_scaling =
+            output_width != input_config.width || output_height != input_config.height;
+
+        if needs_scaling {
+            debug!(
+                "Scaling video frames for H264 encoding from {}x{} to {}x{}",
+                input_config.width, input_config.height, output_width, output_height
+            );
+        }
+
+        let converter = if needs_pixel_conversion || needs_scaling {
+            let flags = if needs_scaling {
+                ffmpeg::software::scaling::flag::Flags::BICUBIC
+            } else {
+                ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR
+            };
+
+            match ffmpeg::software::scaling::Context::get(
+                input_config.pixel_format,
+                input_config.width,
+                input_config.height,
+                output_format,
+                output_width,
+                output_height,
+                flags,
+            ) {
+                Ok(context) => Some(context),
+                Err(e) => {
+                    if needs_pixel_conversion {
                         error!(
-                            "Failed to create converter from {:?} to NV12: {:?}",
-                            input_config.pixel_format, e
+                            "Failed to create converter from {:?} to {:?}: {:?}",
+                            input_config.pixel_format, output_format, e
                         );
-                        H264EncoderError::PixFmtNotSupported(input_config.pixel_format)
-                    })?,
-                ),
-            )
+                        return Err(H264EncoderError::PixFmtNotSupported(
+                            input_config.pixel_format,
+                        ));
+                    }
+
+                    return Err(H264EncoderError::FFmpeg(e));
+                }
+            }
         } else {
-            (input_config.pixel_format, None)
+            None
         };
 
         let mut encoder_ctx = context::Context::new_with_codec(codec);
@@ -105,16 +157,16 @@ impl H264EncoderBuilder {
         encoder_ctx.set_threading(Config::count(thread_count));
         let mut encoder = encoder_ctx.encoder().video()?;
 
-        encoder.set_width(input_config.width);
-        encoder.set_height(input_config.height);
-        encoder.set_format(format);
+        encoder.set_width(output_width);
+        encoder.set_height(output_height);
+        encoder.set_format(output_format);
         encoder.set_time_base(input_config.time_base);
         encoder.set_frame_rate(Some(input_config.frame_rate));
 
         // let target_bitrate = compression.bitrate();
         let bitrate = get_bitrate(
-            input_config.width,
-            input_config.height,
+            output_width,
+            output_height,
             input_config.frame_rate.0 as f32 / input_config.frame_rate.1 as f32,
             self.bpp,
         );
@@ -133,17 +185,21 @@ impl H264EncoderBuilder {
         Ok(H264Encoder {
             base: EncoderBase::new(stream_index),
             encoder,
-            config: self.input_config,
             converter,
+            output_format,
+            output_width,
+            output_height,
         })
     }
 }
 
 pub struct H264Encoder {
     base: EncoderBase,
     encoder: encoder::Video,
-    config: VideoInfo,
     converter: Option<ffmpeg::software::scaling::Context>,
+    output_format: format::Pixel,
+    output_width: u32,
+    output_height: u32,
 }
 
 #[derive(thiserror::Error, Debug)]
@@ -170,16 +226,16 @@ impl H264Encoder {
         self.base
             .update_pts(&mut frame, timestamp, &mut self.encoder);
 
-        let frame = if let Some(converter) = &mut self.converter {
-            let mut new_frame = frame::Video::empty();
+        if let Some(converter) = &mut self.converter {
+            let pts = frame.pts();
+            let mut converted =
+                frame::Video::new(self.output_format, self.output_width, self.output_height);
             converter
-                .run(&frame, &mut new_frame)
+                .run(&frame, &mut converted)
                 .map_err(QueueFrameError::Converter)?;
-            new_frame.set_pts(frame.pts());
-            new_frame
-        } else {
-            frame
-        };
+            converted.set_pts(pts);
+            frame = converted;
+        }
 
         self.base
             .send_frame(&frame, output, &mut self.encoder)

crates/recording/src/output_pipeline/win.rs

@@ -103,7 +103,19 @@ impl Muxer for WindowsMuxer {
                             error!("Failed to create native encoder: {e}");
                             info!("Falling back to software H264 encoder");
 
+                            let fallback_width = if output_size.Width > 0 {
+                                output_size.Width as u32
+                            } else {
+                                video_config.width
+                            };
+                            let fallback_height = if output_size.Height > 0 {
+                                output_size.Height as u32
+                            } else {
+                                video_config.height
+                            };
+
                             cap_enc_ffmpeg::H264Encoder::builder(video_config)
+                                .with_output_size(fallback_width, fallback_height)
                                 .build(&mut output)
                                 .map(either::Right)
                                 .map_err(|e| anyhow!("ScreenSoftwareEncoder/{e}"))