From d53022410cb8620c495865e78e2cfe1ba6ead5e2 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 11 Sep 2025 13:45:02 +1000 Subject: [PATCH 1/4] Fix connectivity issue where sdpMid and sdpMLineIdx were causing connections to fail, these can be safely dropped as we use bundle by default and therefore these attributes are not used. --- .../WebRtcPlayer/WebRtcPlayerController.ts | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index 1241a5cee..7fbc02b07 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -1106,19 +1106,35 @@ export class WebRtcPlayerController { this.pixelStreaming._onLatencyCalculated(latencyInfo); }; - /* When the Peer Connection wants to send an offer have it handled */ + /* When our PeerConnection sends want to send an offer call our handler */ this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => { this.handleSendWebRTCOffer(offer); }; - /* Set event handler for when local answer description is set */ - this.peerConnectionController.onSetLocalDescription = (answer: RTCSessionDescriptionInit) => { - this.handleSendWebRTCAnswer(answer); + /* Set event handler for when local description is set */ + this.peerConnectionController.onSetLocalDescription = (sdp: RTCSessionDescriptionInit) => { + if (sdp.type === 'offer') { + this.handleSendWebRTCOffer(sdp); + } else if (sdp.type === 'answer') { + this.handleSendWebRTCAnswer(sdp); + } else { + Logger.Error( + `PeerConnectionController onSetLocalDescription was called with unexpected type ${sdp.type}` + ); + } }; - /* Set event handler for when remote offer description is set */ - this.peerConnectionController.onSetRemoteDescription = (offer: RTCSessionDescriptionInit) => { - this.pixelStreaming._onWebRtcSdpOffer(offer); + /* Event handler for when PeerConnection's remote description is set */ + this.peerConnectionController.onSetRemoteDescription = (sdp: RTCSessionDescriptionInit) => { + if (sdp.type === 'offer') { + this.pixelStreaming._onWebRtcSdpOffer(sdp); + } else if (sdp.type === 'answer') { + this.pixelStreaming._onWebRtcSdpAnswer(sdp); + } else { + Logger.Error( + `PeerConnectionController onSetRemoteDescription was called with unexpected type ${sdp.type}` + ); + } }; /* When the Peer Connection ice candidate is added have it handled */ @@ -1463,14 +1479,22 @@ export class WebRtcPlayerController { } /** - * When an ice Candidate is received from the Signaling server add it to the Peer Connection Client - * @param iceCandidate - Ice Candidate from Server + * Handler for when a remote ICE candidate is received. + * @param iceCandidateInit - Initialization data used to make the actual ICE Candidate. */ - handleIceCandidate(iceCandidate: RTCIceCandidateInit) { - Logger.Info('Web RTC Controller: onWebRtcIce'); + handleIceCandidate(iceCandidateInit: RTCIceCandidateInit) { + Logger.Info(`Remote ICE candidate information received: ${JSON.stringify(iceCandidateInit)}`); + + // We are using "bundle" policy for media lines so we remove the sdpMid and sdpMLineIndex attributes + // from ICE candidates as these are legacy attributes for when bundle is not used. + // If we don't do this the browser may be unable to form a media connection + // because some browsers are brittle if the bundle master (e.g. commonly mid=0) doesn't get a candidate first. + const remoteIceCandidate = new RTCIceCandidate({ + candidate: iceCandidateInit.candidate, + sdpMid: '' + }); - const candidate = new RTCIceCandidate(iceCandidate); - this.peerConnectionController.handleOnIce(candidate); + this.peerConnectionController.handleOnIce(remoteIceCandidate); } /** @@ -1478,8 +1502,8 @@ export class WebRtcPlayerController { * @param iceEvent - RTC Peer ConnectionIceEvent) { */ handleSendIceCandidate(iceEvent: RTCPeerConnectionIceEvent) { - Logger.Info('OnIceCandidate'); if (iceEvent.candidate && iceEvent.candidate.candidate) { + Logger.Info(`Local ICE candidate generated: ` + JSON.stringify(iceEvent.candidate)); this.protocol.sendMessage( MessageHelpers.createMessage(Messages.iceCandidate, { candidate: iceEvent.candidate }) ); @@ -1504,6 +1528,13 @@ export class WebRtcPlayerController { * @param offer - RTC Session Description */ handleSendWebRTCOffer(offer: RTCSessionDescriptionInit) { + if (offer.type !== 'offer') { + Logger.Error( + `handleSendWebRTCOffer was called with type ${offer.type} - it only expects "offer"` + ); + return; + } + Logger.Info('Sending the offer to the Server'); const extraParams = { @@ -1513,6 +1544,9 @@ export class WebRtcPlayerController { }; this.protocol.sendMessage(MessageHelpers.createMessage(Messages.offer, extraParams)); + + // Send offer back to Pixel Streaming main class for event dispatch + this.pixelStreaming._onWebRtcSdpOffer(offer); } /** @@ -1520,6 +1554,13 @@ export class WebRtcPlayerController { * @param answer - RTC Session Description */ handleSendWebRTCAnswer(answer: RTCSessionDescriptionInit) { + if (answer.type !== 'answer') { + Logger.Error( + `handleSendWebRTCAnswer was called with type ${answer.type} - it only expects "answer"` + ); + return; + } + Logger.Info('Sending the answer to the Server'); const extraParams = { From 0d53477d04a6f3c8061e8642dd60f5d9771434f1 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 11 Sep 2025 13:47:28 +1000 Subject: [PATCH 2/4] Fix comment --- Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index 7fbc02b07..ed14d6ef5 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -1106,7 +1106,7 @@ export class WebRtcPlayerController { this.pixelStreaming._onLatencyCalculated(latencyInfo); }; - /* When our PeerConnection sends want to send an offer call our handler */ + /* When our PeerConnection wants to send an offer call our handler */ this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => { this.handleSendWebRTCOffer(offer); }; From 96066f30d3abe8a7ae05c9dc7357223052ca6bb2 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 11 Sep 2025 14:31:32 +1000 Subject: [PATCH 3/4] Fix up tests to match the new requirements about ice candidate stripping --- .../PeerConnectionController/PeerConnectionController.ts | 2 +- .../library/src/PixelStreaming/PixelStreaming.test.ts | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 4a4b259b8..687ffb6af 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -144,7 +144,7 @@ export class PeerConnectionController { return this.peerConnection?.setLocalDescription(Answer); }) .then(() => { - this.onSetLocalDescription(this.peerConnection?.currentLocalDescription); + this.onSetLocalDescription(this.peerConnection?.localDescription); }) .catch((err) => { Logger.Error(`createAnswer() failed - ${err}`); diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts index 004711d1c..bdaba258a 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts @@ -322,7 +322,14 @@ describe('PixelStreaming', () => { triggerSdpOfferMessage(); triggerIceCandidateMessage(); - expect(rtcPeerConnectionSpyFunctions.addIceCandidateSpy).toHaveBeenCalledWith(iceCandidate) + // Expect ice candidate to be stripped even if passed in with sdpMid and sdpMLineIndex + // as these values are not required when using bundle (which we assume) + const strippedIceCandidate = new RTCIceCandidate({ + candidate: iceCandidate.candidate, + sdpMid: "" + }); + + expect(rtcPeerConnectionSpyFunctions.addIceCandidateSpy).toHaveBeenCalledWith(strippedIceCandidate) }); it('should emit webRtcConnected event when ICE connection state is connected', () => { From 41bfea8f2ea315c7ca6804ecd2b14c2dfa8489bb Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 11 Sep 2025 14:55:59 +1000 Subject: [PATCH 4/4] Added changeset explaining this fix --- .changeset/rich-sites-hear.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/rich-sites-hear.md diff --git a/.changeset/rich-sites-hear.md b/.changeset/rich-sites-hear.md new file mode 100644 index 000000000..38acc68d7 --- /dev/null +++ b/.changeset/rich-sites-hear.md @@ -0,0 +1,5 @@ +--- +'@epicgames-ps/lib-pixelstreamingfrontend-ue5.6': minor +--- + +This change fixes an intermittent WebRTC connection failure where even when the appropriate ICE candidates were present the conection would sometimes fail to be made. This was caused due to the order that ICE candidates were being sent (hence the intermittent nature of the issues) and the fact that ICE candidates sent from Pixel Streaming plugin contain sdpMid and sdpMLineIndex. sdpMid and sdpMLineIndex are only necessary in legacy, non bundle, WebRTC streams; however, Pixel Streaming always assumes bundle is used and these attributes can safely be set to empty strings/omitted (respectively). We perform this modification in the frontend library prior to adding the ICE candidate to the peer connection. This change was tested on a wide range of target devices and browsers to ensure there was no adverse side effects prior.