audio works with no autosampling

Author: Roy3838

app/mobile/Cargo.lock

@@ -32,6 +32,7 @@ tauri-plugin-deep-link = "2"
 tauri-plugin-web-auth = "1"
 tauri-plugin-iap = "0.7"
 tauri-plugin-opener = "2"
+tauri-plugin-log = "2"
 
 # HTTP server for broadcast extension and proxy
 tokio = { version = "1", features = ["full"] }

app/mobile/Cargo.toml

@@ -162,6 +162,8 @@ class SampleHandler: RPBroadcastSampleHandler {
         AudioRingBuffer.shared.write(samples: samples, sampleRate: sampleRate, timestamp: timestamp)
     }
 
+    private var audioFormatLogged = false
+
     private func audioBufferToFloats(_ sampleBuffer: CMSampleBuffer) -> [Float]? {
         var audioBufferList = AudioBufferList()
         var blockBuffer: CMBlockBuffer?
@@ -190,33 +192,121 @@ class SampleHandler: RPBroadcastSampleHandler {
 
         let sourceFormat = streamDescription.pointee.mFormatFlags
         let bytesPerSample = Int(streamDescription.pointee.mBitsPerChannel / 8)
-        let numSamples = dataSize / bytesPerSample
+        let channelCount = Int(streamDescription.pointee.mChannelsPerFrame)
+        let sampleRate = streamDescription.pointee.mSampleRate
+        let bytesPerFrame = Int(streamDescription.pointee.mBytesPerFrame)
+        let framesPerPacket = streamDescription.pointee.mFramesPerPacket
+        let isFloat = (sourceFormat & kAudioFormatFlagIsFloat) != 0
+        let isInterleaved = (sourceFormat & kAudioFormatFlagIsNonInterleaved) == 0
+
+        // Log format once for debugging
+        if !audioFormatLogged {
+            audioFormatLogged = true
+            DebugLog.shared.log("🎵 Audio Format Debug:")
+            DebugLog.shared.log("   - Sample Rate: \(sampleRate) Hz")
+            DebugLog.shared.log("   - Channels: \(channelCount)")
+            DebugLog.shared.log("   - Bits/Channel: \(streamDescription.pointee.mBitsPerChannel)")
+            DebugLog.shared.log("   - Bytes/Sample: \(bytesPerSample)")
+            DebugLog.shared.log("   - Bytes/Frame: \(bytesPerFrame)")
+            DebugLog.shared.log("   - Frames/Packet: \(framesPerPacket)")
+            DebugLog.shared.log("   - Is Float: \(isFloat)")
+            DebugLog.shared.log("   - Is Interleaved: \(isInterleaved)")
+            DebugLog.shared.log("   - Data size: \(dataSize) bytes")
+            DebugLog.shared.log("   - Format flags: 0x\(String(sourceFormat, radix: 16))")
+
+            // Calculate frames for this first chunk
+            let firstNumFrames = dataSize / bytesPerFrame
+            DebugLog.shared.log("   - Frames per chunk: \(firstNumFrames)")
+            DebugLog.shared.log("   - Output: \(firstNumFrames) mono f32 samples (\(firstNumFrames * 4) bytes)")
+            if channelCount == 2 {
+                DebugLog.shared.log("   ✅ Stereo-to-mono downmix enabled")
+            }
+        }
 
-        // Convert to f32 PCM
-        var floatData = [Float](repeating: 0, count: numSamples)
+        // Calculate number of FRAMES (a frame contains samples for all channels)
+        let numFrames = dataSize / bytesPerFrame
+
+        // Output is MONO - one float per frame (downmix stereo if needed)
+        var floatData = [Float](repeating: 0, count: numFrames)
 
         // Check if source is already float
-        if (sourceFormat & kAudioFormatFlagIsFloat) != 0 {
-            // Already float, just copy
-            if bytesPerSample == 4 {
-                mData.withMemoryRebound(to: Float.self, capacity: numSamples) { ptr in
-                    floatData = Array(UnsafeBufferPointer(start: ptr, count: numSamples))
+        if isFloat {
+            if channelCount == 1 {
+                // Mono float - direct copy
+                mData.withMemoryRebound(to: Float.self, capacity: numFrames) { ptr in
+                    floatData = Array(UnsafeBufferPointer(start: ptr, count: numFrames))
+                }
+            } else if channelCount == 2 && isInterleaved {
+                // Stereo float interleaved - downmix to mono
+                mData.withMemoryRebound(to: Float.self, capacity: numFrames * 2) { ptr in
+                    for i in 0..<numFrames {
+                        let left = ptr[i * 2]
+                        let right = ptr[i * 2 + 1]
+                        floatData[i] = (left + right) * 0.5
+                    }
                 }
             }
         } else {
-            // Convert from integer to float
+            // Integer format - convert to float
+            // Check endianness from format flags
+            let isBigEndian = (sourceFormat & kAudioFormatFlagIsBigEndian) != 0
+
             if bytesPerSample == 2 {
-                // Int16 to Float
-                mData.withMemoryRebound(to: Int16.self, capacity: numSamples) { ptr in
-                    for i in 0..<numSamples {
-                        floatData[i] = Float(ptr[i]) / Float(Int16.max)
+                // Int16 format
+                if channelCount == 1 {
+                    // Mono Int16
+                    mData.withMemoryRebound(to: Int16.self, capacity: numFrames) { ptr in
+                        for i in 0..<numFrames {
+                            var sample = ptr[i]
+                            if isBigEndian {
+                                sample = Int16(bigEndian: sample)
+                            }
+                            floatData[i] = Float(sample) / Float(Int16.max)
+                        }
+                    }
+                } else if channelCount == 2 && isInterleaved {
+                    // Stereo Int16 interleaved - downmix to mono
+                    mData.withMemoryRebound(to: Int16.self, capacity: numFrames * 2) { ptr in
+                        for i in 0..<numFrames {
+                            var leftRaw = ptr[i * 2]
+                            var rightRaw = ptr[i * 2 + 1]
+                            if isBigEndian {
+                                leftRaw = Int16(bigEndian: leftRaw)
+                                rightRaw = Int16(bigEndian: rightRaw)
+                            }
+                            let left = Float(leftRaw) / Float(Int16.max)
+                            let right = Float(rightRaw) / Float(Int16.max)
+                            floatData[i] = (left + right) * 0.5
+                        }
                     }
                 }
             } else if bytesPerSample == 4 {
-                // Int32 to Float
-                mData.withMemoryRebound(to: Int32.self, capacity: numSamples) { ptr in
-                    for i in 0..<numSamples {
-                        floatData[i] = Float(ptr[i]) / Float(Int32.max)
+                // Int32 format
+                if channelCount == 1 {
+                    // Mono Int32
+                    mData.withMemoryRebound(to: Int32.self, capacity: numFrames) { ptr in
+                        for i in 0..<numFrames {
+                            var sample = ptr[i]
+                            if isBigEndian {
+                                sample = Int32(bigEndian: sample)
+                            }
+                            floatData[i] = Float(sample) / Float(Int32.max)
+                        }
+                    }
+                } else if channelCount == 2 && isInterleaved {
+                    // Stereo Int32 interleaved - downmix to mono
+                    mData.withMemoryRebound(to: Int32.self, capacity: numFrames * 2) { ptr in
+                        for i in 0..<numFrames {
+                            var leftRaw = ptr[i * 2]
+                            var rightRaw = ptr[i * 2 + 1]
+                            if isBigEndian {
+                                leftRaw = Int32(bigEndian: leftRaw)
+                                rightRaw = Int32(bigEndian: rightRaw)
+                            }
+                            let left = Float(leftRaw) / Float(Int32.max)
+                            let right = Float(rightRaw) / Float(Int32.max)
+                            floatData[i] = (left + right) * 0.5
+                        }
                     }
                 }
             }

app/mobile/gen/apple/broadcast/SampleHandler.swift

@@ -129,7 +129,7 @@ class VideoFrameBuffer {
 
         // Write JPEG data to buffer section
         let dataStart = data.advanced(by: headerSize)
-        jpegData.withUnsafeBytes { srcBuffer in
+        _ = jpegData.withUnsafeBytes { srcBuffer in
             memcpy(dataStart, srcBuffer.baseAddress!, Int(frameSize))
         }
 

app/mobile/gen/apple/broadcast/VideoFrameBuffer.swift

@@ -2852,6 +2852,24 @@
           "const": "ios-keyboard:deny-ping",
           "markdownDescription": "Denies the ping command without any pre-configured scope."
         },
+        {
+          "description": "Allows the log command\n#### This default permission set includes:\n\n- `allow-log`",
+          "type": "string",
+          "const": "log:default",
+          "markdownDescription": "Allows the log command\n#### This default permission set includes:\n\n- `allow-log`"
+        },
+        {
+          "description": "Enables the log command without any pre-configured scope.",
+          "type": "string",
+          "const": "log:allow-log",
+          "markdownDescription": "Enables the log command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the log command without any pre-configured scope.",
+          "type": "string",
+          "const": "log:deny-log",
+          "markdownDescription": "Denies the log command without any pre-configured scope."
+        },
         {
           "description": "This permission set configures which\nnotification features are by default exposed.\n\n#### Granted Permissions\n\nIt allows all notification related features.\n\n\n#### This default permission set includes:\n\n- `allow-is-permission-granted`\n- `allow-request-permission`\n- `allow-notify`\n- `allow-register-action-types`\n- `allow-register-listener`\n- `allow-cancel`\n- `allow-get-pending`\n- `allow-remove-active`\n- `allow-get-active`\n- `allow-check-permissions`\n- `allow-show`\n- `allow-batch`\n- `allow-list-channels`\n- `allow-delete-channel`\n- `allow-create-channel`\n- `allow-permission-state`",
           "type": "string",

app/mobile/gen/schemas/acl-manifests.json

@@ -2852,6 +2852,24 @@
           "const": "ios-keyboard:deny-ping",
           "markdownDescription": "Denies the ping command without any pre-configured scope."
         },
+        {
+          "description": "Allows the log command\n#### This default permission set includes:\n\n- `allow-log`",
+          "type": "string",
+          "const": "log:default",
+          "markdownDescription": "Allows the log command\n#### This default permission set includes:\n\n- `allow-log`"
+        },
+        {
+          "description": "Enables the log command without any pre-configured scope.",
+          "type": "string",
+          "const": "log:allow-log",
+          "markdownDescription": "Enables the log command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the log command without any pre-configured scope.",
+          "type": "string",
+          "const": "log:deny-log",
+          "markdownDescription": "Denies the log command without any pre-configured scope."
+        },
         {
           "description": "This permission set configures which\nnotification features are by default exposed.\n\n#### Granted Permissions\n\nIt allows all notification related features.\n\n\n#### This default permission set includes:\n\n- `allow-is-permission-granted`\n- `allow-request-permission`\n- `allow-notify`\n- `allow-register-action-types`\n- `allow-register-listener`\n- `allow-cancel`\n- `allow-get-pending`\n- `allow-remove-active`\n- `allow-get-active`\n- `allow-check-permissions`\n- `allow-show`\n- `allow-batch`\n- `allow-list-channels`\n- `allow-delete-channel`\n- `allow-create-channel`\n- `allow-permission-state`",
           "type": "string",