From b9e12d2664facf6beac097522238c7e66b225062 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 6 Dec 2024 17:38:19 +1000 Subject: [PATCH 01/27] Add sending abs-capture-time in JS streamer and a test --- Extras/FrontendTests/tests/fixtures.ts | 19 +++++-- .../tests/peerconnection.spec.ts | 40 ++++++++++++++ Extras/JSStreamer/src/streamer.ts | 55 +++++++++++++++---- 3 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 Extras/FrontendTests/tests/peerconnection.spec.ts diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index fa1e4127b..55da4b5bb 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -3,25 +3,32 @@ import { test as base, Page } from '@playwright/test'; type PSTestFixtures = { streamerPage: Page; streamerId: string; + localDescription: RTCSessionDescriptionInit; }; export const test = base.extend({ streamerPage: async ({ context }, use) => { const streamerPage = await context.newPage(); - await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}?SignallingURL=${process.env.STREAMER_SIGNALLING_URL}`); + await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}` + `${process.env.STREAMER_SIGNALLING_URL !== undefined ? '?SignallingURL=' + process.env.STREAMER_SIGNALLING_URL : ""}`); await use(streamerPage); }, streamerId: async ({ streamerPage }, use) => { - const idPromise: Promise = streamerPage.evaluate(()=> { - return new Promise((resolve) => { + + const idPromise: Promise = new Promise(async (resolve)=> { + + // Expose the resolve function to the browser context + await streamerPage.exposeFunction('resolveFromIdPromise', resolve); + + streamerPage.evaluate(()=> { window.streamer.on('endpoint_id_confirmed', () => { - resolve(window.streamer.id); + window.resolveFromIdPromise(window.streamer.id); }); - }) + }); }); + await streamerPage.getByText('Start Streaming').click(); const streamerId: string = await idPromise; await use(streamerId); - }, + } }); diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts new file mode 100644 index 000000000..38c7da7da --- /dev/null +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -0,0 +1,40 @@ +import { test } from './fixtures'; +import { expect } from './matchers'; +import * as helpers from './helpers'; + +test('Test abs-capture-time header extension found turned on for streamer', { + tag: ['@capture-time'], +}, async ({ page, streamerPage, streamerId }) => { + + // // helps debugging + // helpers.attachToConsoleEvents(streamerPage, (...args: any[]) => { + // console.log("Streamer: ", ...args); + // }); + // + // helpers.attachToConsoleEvents(page, (...args: any[]) => { + // console.log("Player: ", ...args); + // }); + + const localDescription: Promise = new Promise(async (resolve) => { + + // Expose the resolve function to the browser context + await streamerPage.exposeFunction('resolveFromLocalDescriptionPromise', resolve); + + streamerPage.evaluate(() => { + window.streamer.on('local_description_set', (localDescription: RTCSessionDescriptionInit) => { + resolveFromLocalDescriptionPromise(localDescription); + }); + }); + }); + + await page.goto(`/?StreamerId=${streamerId}`); + await page.getByText('Click to start').click(); + + await helpers.waitForVideo(page); + let localDescSdp: RTCSessionDescriptionInit = await localDescription; + + expect(localDescSdp.sdp).toBeDefined(); + + // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer + expect(localDescSdp.sdp).toContain("abs-capture-time"); +}); diff --git a/Extras/JSStreamer/src/streamer.ts b/Extras/JSStreamer/src/streamer.ts index 6dc5dc63e..a9853480e 100644 --- a/Extras/JSStreamer/src/streamer.ts +++ b/Extras/JSStreamer/src/streamer.ts @@ -52,6 +52,9 @@ export class PlayerPeer { const protocolVersion = '1.0.0'; +// Official uri for abs-capture-time RTP header extension, used to signal we want to use this extension. +const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; + export class Streamer extends EventEmitter { id: string; settings: Settings; @@ -129,7 +132,9 @@ export class Streamer extends EventEmitter { } handleConfigMessage(msg: Messages.config) { - this.peerConnectionOptions = msg.peerConnectionOptions; + if(msg.peerConnectionOptions !== undefined) { + this.peerConnectionOptions = msg.peerConnectionOptions; + } } handleIdentifyMessage(_msg: Messages.identify) { @@ -213,6 +218,14 @@ export class Streamer extends EventEmitter { peerConnection .createOffer() .then((offer) => { + + if(offer.sdp == undefined) { + return; + } + + // Munge offer + offer.sdp = this.mungeOffer(offer.sdp); + peerConnection .setLocalDescription(offer) .then(() => { @@ -222,6 +235,7 @@ export class Streamer extends EventEmitter { sdp: offer.sdp }) ); + this.emit('local_description_set', offer); }) .catch(() => {}); }) @@ -232,8 +246,8 @@ export class Streamer extends EventEmitter { peerConnection .getStats() .then((stats: RTCStatsReport) => { - let qpSum: number; - let fps: number; + let qpSum: number | undefined = undefined; + let fps: number | undefined = undefined; stats.forEach((report) => { /* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment */ if (report.type == 'outbound-rtp' && report.mediaType == 'video') { @@ -243,14 +257,12 @@ export class Streamer extends EventEmitter { /* eslint-enable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment */ }); const nowTime = Date.now(); - if (newPlayer.lastStatsTime) { + if (newPlayer.lastStatsTime != undefined && newPlayer.lastQpSum !== undefined && qpSum !== undefined && fps !== undefined) { const deltaMillis = nowTime - newPlayer.lastStatsTime; const qpDelta = (qpSum - newPlayer.lastQpSum) * (deltaMillis / 1000); const qpAvg = qpDelta / fps; - newPlayer.dataChannel.send( - this.constructMessage(DataProtocol.FromStreamer.VideoEncoderAvgQP, qpAvg) - ); + newPlayer.dataChannel.send(this.constructMessage(DataProtocol.FromStreamer.VideoEncoderAvgQP, qpAvg)); } newPlayer.lastQpSum = qpSum; newPlayer.lastStatsTime = nowTime; @@ -291,6 +303,30 @@ export class Streamer extends EventEmitter { } } + mungeOffer(offerSDP: string) : string { + // Add the abs-capture-time header extension to the sdp extmap + return this.addHeaderExtensionToSdp(offerSDP, kAbsCaptureTime); + } + + addHeaderExtensionToSdp(sdp: string, uri: string) : string { + // Find the highest used header extension id by sorting the extension ids used, + // eliminating duplicates and adding one. + // Todo: Update this when WebRTC in Chrome supports the header extension API. + const usedIds = sdp.split('\n') + .filter(line => line.startsWith('a=extmap:')) + .map(line => parseInt(line.split(' ')[0].substring(9), 10)) + .sort((a, b) => a - b) + .filter((item, index, array) => array.indexOf(item) === index); + const nextId = usedIds[usedIds.length - 1] + 1; + const extmapLine = 'a=extmap:' + nextId + ' ' + uri + '\r\n'; + + const sections = sdp.split('\nm=').map((part, index) => { + return (index > 0 ? 'm=' + part : part).trim() + '\r\n'; + }); + const sessionPart = sections.shift(); + return sessionPart + sections.map(mediaSection => mediaSection + extmapLine).join(''); + } + sendDataProtocol(playerId: string) { const playerPeer = this.playerMap[playerId]; if (playerPeer) { @@ -342,10 +378,7 @@ export class Streamer extends EventEmitter { let argIndex = 0; if (messageDef.structure.length != args.length) { - console.log( - `Incorrect number of parameters given to constructMessage. Got ${args.length}, expected ${messageDef.structure.length}` - ); - return null; + throw new Error(`Incorrect number of parameters given to constructMessage. Got ${args.length}, expected ${messageDef.structure.length}`); } dataSize += 1; // message type From fc83cee56b4881edda89069aa15c5518789d56df Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 6 Dec 2024 17:43:16 +1000 Subject: [PATCH 02/27] Add some comments --- Extras/FrontendTests/tests/fixtures.ts | 4 ++++ Extras/FrontendTests/tests/peerconnection.spec.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index 55da4b5bb..77b27fdf6 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -19,6 +19,10 @@ export const test = base.extend({ // Expose the resolve function to the browser context await streamerPage.exposeFunction('resolveFromIdPromise', resolve); + // Note: If page.evaluate is passed a promise it will try to await it immediately + // to avoid this hanging here waiting for endpoint_id_confirmed we instead + // wrap the page.evaluate in a promise and expose the resolve into the streamer page + // to be called when the endpoint_id_confirmed is actually called. streamerPage.evaluate(()=> { window.streamer.on('endpoint_id_confirmed', () => { window.resolveFromIdPromise(window.streamer.id); diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 38c7da7da..1f389e372 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -2,7 +2,7 @@ import { test } from './fixtures'; import { expect } from './matchers'; import * as helpers from './helpers'; -test('Test abs-capture-time header extension found turned on for streamer', { +test('Test abs-capture-time header extension found for streamer', { tag: ['@capture-time'], }, async ({ page, streamerPage, streamerId }) => { From b950af3c910cef0c6d169ab5e15c1060657c3e1e Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Tue, 10 Dec 2024 15:02:28 +1000 Subject: [PATCH 03/27] Added extra commenting --- Extras/FrontendTests/tests/fixtures.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index 77b27fdf6..37986ca6d 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -21,7 +21,7 @@ export const test = base.extend({ // Note: If page.evaluate is passed a promise it will try to await it immediately // to avoid this hanging here waiting for endpoint_id_confirmed we instead - // wrap the page.evaluate in a promise and expose the resolve into the streamer page + // wrap the page.evaluate in a promise and expose the resolve argument/function into the streamer page // to be called when the endpoint_id_confirmed is actually called. streamerPage.evaluate(()=> { window.streamer.on('endpoint_id_confirmed', () => { From 47398a8e064e16580f215f693e483ec2211e2c63 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Tue, 10 Dec 2024 15:45:26 +1000 Subject: [PATCH 04/27] Added SDPUtils to /Common --- Common/src/Util/SdpUtils.ts | 24 +++++++++++++++++++++++ Common/src/pixelstreamingcommon.ts | 1 + Extras/JSStreamer/src/streamer.ts | 28 ++++----------------------- Frontend/library/src/Config/Config.ts | 14 ++++++++++++++ 4 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 Common/src/Util/SdpUtils.ts diff --git a/Common/src/Util/SdpUtils.ts b/Common/src/Util/SdpUtils.ts new file mode 100644 index 000000000..7d6ade9cd --- /dev/null +++ b/Common/src/Util/SdpUtils.ts @@ -0,0 +1,24 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +export class SDPUtils { + + static addHeaderExtensionToSdp(sdp: string, uri: string) : string { + // Find the highest used header extension id by sorting the extension ids used, + // eliminating duplicates and adding one. + // Todo: Update this when WebRTC in Chrome supports the header extension API. + const usedIds = sdp.split('\n') + .filter(line => line.startsWith('a=extmap:')) + .map(line => parseInt(line.split(' ')[0].substring(9), 10)) + .sort((a, b) => a - b) + .filter((item, index, array) => array.indexOf(item) === index); + const nextId = usedIds[usedIds.length - 1] + 1; + const extmapLine = 'a=extmap:' + nextId + ' ' + uri + '\r\n'; + + const sections = sdp.split('\nm=').map((part, index) => { + return (index > 0 ? 'm=' + part : part).trim() + '\r\n'; + }); + const sessionPart = sections.shift(); + return sessionPart + sections.map(mediaSection => mediaSection + extmapLine).join(''); + } + +} \ No newline at end of file diff --git a/Common/src/pixelstreamingcommon.ts b/Common/src/pixelstreamingcommon.ts index d498f455b..4aa62437a 100644 --- a/Common/src/pixelstreamingcommon.ts +++ b/Common/src/pixelstreamingcommon.ts @@ -9,3 +9,4 @@ export { EventEmitter } from './Event/EventEmitter'; export { MessageRegistry } from './Messages/message_registry'; export * as Messages from './Messages/signalling_messages'; export * as MessageHelpers from './Messages/message_helpers'; +export * from './Util/SdpUtils'; diff --git a/Extras/JSStreamer/src/streamer.ts b/Extras/JSStreamer/src/streamer.ts index a9853480e..8b79bf34a 100644 --- a/Extras/JSStreamer/src/streamer.ts +++ b/Extras/JSStreamer/src/streamer.ts @@ -5,7 +5,8 @@ import { Messages, MessageHelpers, BaseMessage, - EventEmitter + EventEmitter, + SDPUtils } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; import { DataProtocol } from './protocol'; @@ -52,9 +53,6 @@ export class PlayerPeer { const protocolVersion = '1.0.0'; -// Official uri for abs-capture-time RTP header extension, used to signal we want to use this extension. -const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; - export class Streamer extends EventEmitter { id: string; settings: Settings; @@ -305,26 +303,8 @@ export class Streamer extends EventEmitter { mungeOffer(offerSDP: string) : string { // Add the abs-capture-time header extension to the sdp extmap - return this.addHeaderExtensionToSdp(offerSDP, kAbsCaptureTime); - } - - addHeaderExtensionToSdp(sdp: string, uri: string) : string { - // Find the highest used header extension id by sorting the extension ids used, - // eliminating duplicates and adding one. - // Todo: Update this when WebRTC in Chrome supports the header extension API. - const usedIds = sdp.split('\n') - .filter(line => line.startsWith('a=extmap:')) - .map(line => parseInt(line.split(' ')[0].substring(9), 10)) - .sort((a, b) => a - b) - .filter((item, index, array) => array.indexOf(item) === index); - const nextId = usedIds[usedIds.length - 1] + 1; - const extmapLine = 'a=extmap:' + nextId + ' ' + uri + '\r\n'; - - const sections = sdp.split('\nm=').map((part, index) => { - return (index > 0 ? 'm=' + part : part).trim() + '\r\n'; - }); - const sessionPart = sections.shift(); - return sessionPart + sections.map(mediaSection => mediaSection + extmapLine).join(''); + const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; + return SDPUtils.addHeaderExtensionToSdp(offerSDP, kAbsCaptureTime); } sendDataProtocol(playerId: string) { diff --git a/Frontend/library/src/Config/Config.ts b/Frontend/library/src/Config/Config.ts index 9a215e0d1..a44ce5d4a 100644 --- a/Frontend/library/src/Config/Config.ts +++ b/Frontend/library/src/Config/Config.ts @@ -33,6 +33,7 @@ export class Flags { static XRControllerInput = 'XRControllerInput' as const; static WaitForStreamer = 'WaitForStreamer' as const; static HideUI = 'HideUI' as const; + static EnableCaptureTimeExt = 'EnableCaptureTimeExt' as const; } export type FlagsKeys = Exclude; @@ -548,6 +549,19 @@ export class Config { ) ); + this.flags.set( + Flags.EnableCaptureTimeExt, + new SettingFlag( + Flags.EnableCaptureTimeExt, + 'Enable abs-capture-time', + 'Enables the abs-capture-time RTP header extension', + settings && Object.prototype.hasOwnProperty.call(settings, Flags.EnableCaptureTimeExt) + ? settings[Flags.EnableCaptureTimeExt] + : false, + useUrlParams + ) + ); + /** * Numeric parameters */ From 4a3d1316314b113e9bb9083d9a0d1bc2799678d9 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 20 Dec 2024 10:49:43 +1000 Subject: [PATCH 05/27] Working state --- Common/src/Util/SdpUtils.ts | 15 +++++------ Extras/FrontendTests/tests/extras.ts | 13 +++++++++- .../tests/peerconnection.spec.ts | 26 +++++++++++++++++++ .../PeerConnectionController.ts | 7 +++++ .../src/PixelStreaming/PixelStreaming.ts | 12 +++++++-- Frontend/library/src/Util/EventEmitter.ts | 16 ++++++++++++ .../WebRtcPlayer/WebRtcPlayerController.ts | 7 +++-- 7 files changed, 83 insertions(+), 13 deletions(-) diff --git a/Common/src/Util/SdpUtils.ts b/Common/src/Util/SdpUtils.ts index 7d6ade9cd..9393cabf7 100644 --- a/Common/src/Util/SdpUtils.ts +++ b/Common/src/Util/SdpUtils.ts @@ -1,14 +1,14 @@ // Copyright Epic Games, Inc. All Rights Reserved. export class SDPUtils { - - static addHeaderExtensionToSdp(sdp: string, uri: string) : string { + static addHeaderExtensionToSdp(sdp: string, uri: string): string { // Find the highest used header extension id by sorting the extension ids used, // eliminating duplicates and adding one. // Todo: Update this when WebRTC in Chrome supports the header extension API. - const usedIds = sdp.split('\n') - .filter(line => line.startsWith('a=extmap:')) - .map(line => parseInt(line.split(' ')[0].substring(9), 10)) + const usedIds = sdp + .split('\n') + .filter((line) => line.startsWith('a=extmap:')) + .map((line) => parseInt(line.split(' ')[0].substring(9), 10)) .sort((a, b) => a - b) .filter((item, index, array) => array.indexOf(item) === index); const nextId = usedIds[usedIds.length - 1] + 1; @@ -18,7 +18,6 @@ export class SDPUtils { return (index > 0 ? 'm=' + part : part).trim() + '\r\n'; }); const sessionPart = sections.shift(); - return sessionPart + sections.map(mediaSection => mediaSection + extmapLine).join(''); + return sessionPart + sections.map((mediaSection) => mediaSection + extmapLine).join(''); } - -} \ No newline at end of file +} diff --git a/Extras/FrontendTests/tests/extras.ts b/Extras/FrontendTests/tests/extras.ts index e8deb30ea..597f49b3b 100644 --- a/Extras/FrontendTests/tests/extras.ts +++ b/Extras/FrontendTests/tests/extras.ts @@ -1,6 +1,6 @@ import { Page } from 'playwright'; import { Streamer, DataProtocol } from '@epicgames-ps/js-streamer'; -import { PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; +import { PixelStreaming, WebRtcSdpAnswerEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; declare global { interface Window { @@ -82,6 +82,17 @@ export function getCapturedEvents(streamerPage: Page): Promise { + return playerPage.evaluate(() => { + return new Promise((resolve) => { + window.pixelStreaming.addEventListener("webRtcSdpAnswer", (evt : WebRtcSdpAnswerEvent) => { + resolve(evt.data.sdp); + }); + }); + }); +} + export async function getEventsFor(streamerPage: Page, performAction: () => Promise): Promise> { await setupEventCapture(streamerPage); await performAction(); diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 1f389e372..82ddeba65 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -1,6 +1,7 @@ import { test } from './fixtures'; import { expect } from './matchers'; import * as helpers from './helpers'; +import { Flags, WebRtcSdpAnswerEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; test('Test abs-capture-time header extension found for streamer', { tag: ['@capture-time'], @@ -38,3 +39,28 @@ test('Test abs-capture-time header extension found for streamer', { // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer expect(localDescSdp.sdp).toContain("abs-capture-time"); }); + +test('Test abs-capture-time header extension found in player', { + tag: ['@capture-time'], +}, async ({ page, streamerPage, streamerId }) => { + + await page.goto(`/?StreamerId=${streamerId}`); + + await page.waitForLoadState("load"); + + // Enable the flag for the capture extension + await page.evaluate(() => { + window.pixelStreaming.config.setFlagEnabled(Flags.EnableCaptureTimeExt, true); + }); + + await page.getByText('Click to start').click(); + + //const answer: RTCSessionDescriptionInit = await getSdpAnswer(page); + + await helpers.waitForVideo(page); + + //expect(answer.sdp).toBeDefined(); + + // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer + //expect(answer.sdp).toContain("abs-capture-time"); +}); diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index f95a63917..8e252884a 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -6,6 +6,7 @@ import { AggregatedStats } from './AggregatedStats'; import { parseRtpParameters, splitSections } from 'sdp'; import { RTCUtils } from '../Util/RTCUtils'; import { CodecStats } from './CodecStats'; +import { SDPUtils } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; /** * Handles the Peer Connection @@ -237,6 +238,12 @@ export class PeerConnectionController { // We use the line 'useinbandfec=1' (which Opus uses) to set our Opus specific audio parameters. mungedSDP = mungedSDP.replace('useinbandfec=1', audioSDP); + // Add abs-capture-time RTP header extension if we have enabled the setting + if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt)) { + const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; + mungedSDP = SDPUtils.addHeaderExtensionToSdp(mungedSDP, kAbsCaptureTime); + } + return mungedSDP; } diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index cae00bf44..d806963fb 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -29,7 +29,8 @@ import { DataChannelLatencyTestResponseEvent, DataChannelLatencyTestResultEvent, PlayerCountEvent, - WebRtcTCPRelayDetectedEvent + WebRtcTCPRelayDetectedEvent, + WebRtcSdpAnswerEvent } from '../Util/EventEmitter'; import { WebXRController } from '../WebXR/WebXRController'; import { MessageDirection } from '../UeInstanceMessage/StreamMessageController'; @@ -456,12 +457,19 @@ export class PixelStreaming { } /** - * Set up functionality to happen when receiving a webRTC answer + * Set up functionality to happen when SDP negotiation is fully finished. */ _onWebRtcSdp() { this._eventEmitter.dispatchEvent(new WebRtcSdpEvent()); } + /** + * Set up functionality to happen after SDP has been generated and sent. + */ + _onWebRtcSdpAnswer(answer: RTCSessionDescriptionInit) { + this._eventEmitter.dispatchEvent(new WebRtcSdpAnswerEvent(answer)); + } + /** * Emits a StreamLoading event */ diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index 97687066c..145be4ce8 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -90,6 +90,21 @@ export class WebRtcSdpEvent extends Event { } } +/** + * An event that is emitted after the SDP answer is generated and sent. + */ +export class WebRtcSdpAnswerEvent extends Event { + readonly type: 'webRtcSdpAnswer'; + readonly data: { + /** The sdp answer */ + sdp: RTCSessionDescriptionInit; + }; + constructor(answer: RTCSessionDescriptionInit) { + super('webRtcSdpAnswer'); + this.data.sdp = answer; + } +} + /** * An event that is emitted when auto connecting. */ @@ -549,6 +564,7 @@ export type PixelStreamingEvent = | AfkTimedOutEvent | VideoEncoderAvgQPEvent | WebRtcSdpEvent + | WebRtcSdpAnswerEvent | WebRtcAutoConnectEvent | WebRtcConnectingEvent | WebRtcConnectedEvent diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index afe687649..4c9a08742 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -1013,8 +1013,8 @@ export class WebRtcPlayerController { this.handleSendWebRTCOffer(offer); /* When the Peer Connection wants to send an answer have it handled */ - this.peerConnectionController.onSendWebRTCAnswer = (offer: RTCSessionDescriptionInit) => - this.handleSendWebRTCAnswer(offer); + this.peerConnectionController.onSendWebRTCAnswer = (answer: RTCSessionDescriptionInit) => + this.handleSendWebRTCAnswer(answer); /* When the Peer Connection ice candidate is added have it handled */ this.peerConnectionController.onPeerIceCandidate = ( @@ -1375,6 +1375,9 @@ export class WebRtcPlayerController { if (this.isUsingSFU) { this.protocol.sendMessage(MessageHelpers.createMessage(Messages.dataChannelRequest)); } + + // Send answer back to Pixel Streaming main class for event dispatch + this.pixelStreaming._onWebRtcSdpAnswer(answer); } /** From cfcbdf9eb85212ad833dbba337b6c1cc1e7a84d6 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 16 Jan 2025 15:25:52 +1000 Subject: [PATCH 06/27] Added test to check if abs-capture-time is in offer/answer when flag is enabled. --- .../tests/peerconnection.spec.ts | 36 +++++++++++-------- .../PeerConnectionController.ts | 4 +-- Frontend/library/src/Util/EventEmitter.ts | 4 ++- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 82ddeba65..4e892d301 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -1,21 +1,12 @@ import { test } from './fixtures'; import { expect } from './matchers'; import * as helpers from './helpers'; -import { Flags, WebRtcSdpAnswerEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; +import { Flags, PixelStreaming, WebRtcSdpAnswerEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; test('Test abs-capture-time header extension found for streamer', { tag: ['@capture-time'], }, async ({ page, streamerPage, streamerId }) => { - // // helps debugging - // helpers.attachToConsoleEvents(streamerPage, (...args: any[]) => { - // console.log("Streamer: ", ...args); - // }); - // - // helpers.attachToConsoleEvents(page, (...args: any[]) => { - // console.log("Player: ", ...args); - // }); - const localDescription: Promise = new Promise(async (resolve) => { // Expose the resolve function to the browser context @@ -40,7 +31,7 @@ test('Test abs-capture-time header extension found for streamer', { expect(localDescSdp.sdp).toContain("abs-capture-time"); }); -test('Test abs-capture-time header extension found in player', { +test('Test abs-capture-time header extension found in PSInfra frontend', { tag: ['@capture-time'], }, async ({ page, streamerPage, streamerId }) => { @@ -50,17 +41,32 @@ test('Test abs-capture-time header extension found in player', { // Enable the flag for the capture extension await page.evaluate(() => { - window.pixelStreaming.config.setFlagEnabled(Flags.EnableCaptureTimeExt, true); + window.pixelStreaming.config.setFlagEnabled("EnableCaptureTimeExt", true); + }); + + // Wait for the sdp answer + let getSdpAnswer = new Promise(async (resolve) => { + + // Expose the resolve function to the browser context + await page.exposeFunction('resolveFromSdpAnswerPromise', resolve); + + page.evaluate(() => { + window.pixelStreaming.addEventListener("webRtcSdpAnswer", (e: WebRtcSdpAnswerEvent) => { + resolveFromSdpAnswerPromise(e.data.sdp); + }); + }); + }); await page.getByText('Click to start').click(); - //const answer: RTCSessionDescriptionInit = await getSdpAnswer(page); + const answer: RTCSessionDescriptionInit = await getSdpAnswer; await helpers.waitForVideo(page); - //expect(answer.sdp).toBeDefined(); + expect(answer).toBeDefined(); + expect(answer.sdp).toBeDefined(); // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer - //expect(answer.sdp).toContain("abs-capture-time"); + expect(answer.sdp).toContain("abs-capture-time"); }); diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 8e252884a..5a2c911fc 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -127,8 +127,8 @@ export class PeerConnectionController { .then(() => { this.onSendWebRTCAnswer(this.peerConnection?.currentLocalDescription); }) - .catch(() => { - Logger.Error('createAnswer() failed'); + .catch((err) => { + Logger.Error(`createAnswer() failed - ${err}`); }); }); }); diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index 145be4ce8..e0945b0b6 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -101,7 +101,9 @@ export class WebRtcSdpAnswerEvent extends Event { }; constructor(answer: RTCSessionDescriptionInit) { super('webRtcSdpAnswer'); - this.data.sdp = answer; + this.data = { + sdp: answer + }; } } From cf1cf95ba11d9595e867dda709a0392c5d44382f Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 17 Jan 2025 12:07:51 +1000 Subject: [PATCH 07/27] Update JSStreamer dockerfile to build using local files. Speed up build by only installing relevant playwright browsers. --- Extras/FrontendTests/README.md | 10 +- .../dockerfiles/linux/Dockerfile | 5 +- Extras/FrontendTests/package.json | 6 +- Extras/FrontendTests/playwright.config.ts | 2 +- Extras/JSStreamer/Dockerfile | 17 +- Extras/JSStreamer/package.json | 4 +- package-lock.json | 276 +++++------------- 7 files changed, 107 insertions(+), 213 deletions(-) diff --git a/Extras/FrontendTests/README.md b/Extras/FrontendTests/README.md index 74d2c9256..d2d5a0ba3 100755 --- a/Extras/FrontendTests/README.md +++ b/Extras/FrontendTests/README.md @@ -4,13 +4,9 @@ ### Setup ``` npm install -npx playwright install --with-deps -``` - -The above command should install the required browsers but for some reason I find I have to install chrome manually using the following command. - -``` -npx playwright install chrome +npx playwright install-deps +npx playwright install firefox +npx playwright install chromium ``` ### Prepare diff --git a/Extras/FrontendTests/dockerfiles/linux/Dockerfile b/Extras/FrontendTests/dockerfiles/linux/Dockerfile index be60bb780..c2d08b4d5 100644 --- a/Extras/FrontendTests/dockerfiles/linux/Dockerfile +++ b/Extras/FrontendTests/dockerfiles/linux/Dockerfile @@ -4,8 +4,9 @@ WORKDIR /tester COPY /Extras/FrontendTests . RUN npm install -RUN npx playwright install --with-deps -RUN npx playwright install chrome +RUN npx playwright install firefox +RUN npx playwright install chromium +RUN npx playwright install-deps VOLUME /tester/playwright-report diff --git a/Extras/FrontendTests/package.json b/Extras/FrontendTests/package.json index 2d6b445df..e78a481fd 100755 --- a/Extras/FrontendTests/package.json +++ b/Extras/FrontendTests/package.json @@ -4,20 +4,20 @@ "description": "", "main": "index.js", "scripts": { - "test": "playwright test", + "test": "npx playwright test", "build": "" }, "keywords": [], "author": "Epic Games", "license": "MIT", "devDependencies": { - "@playwright/test": "^1.49.0", + "@playwright/test": "^1.49.1", "@types/node": "^20.12.7", "@types/uuid": "^9.0.8" }, "dependencies": { - "@epicgames-ps/lib-pixelstreamingfrontend-ue5.5": "*", "@epicgames-ps/js-streamer": "^0.0.4", + "@epicgames-ps/lib-pixelstreamingfrontend-ue5.5": "*", "dotenv": "^16.4.5", "node-fetch": "^2.7.0", "uuid": "^9.0.0" diff --git a/Extras/FrontendTests/playwright.config.ts b/Extras/FrontendTests/playwright.config.ts index 05c5d4848..429f8e95c 100755 --- a/Extras/FrontendTests/playwright.config.ts +++ b/Extras/FrontendTests/playwright.config.ts @@ -31,7 +31,7 @@ export default defineConfig({ projects: [ { name: 'chrome', - use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + use: { ...devices['Desktop Chrome'], channel: 'chromium' }, }, // { // name: 'chromium', diff --git a/Extras/JSStreamer/Dockerfile b/Extras/JSStreamer/Dockerfile index 04283420d..776852cdd 100644 --- a/Extras/JSStreamer/Dockerfile +++ b/Extras/JSStreamer/Dockerfile @@ -1,9 +1,22 @@ FROM node:20-bookworm +## Note: This dockerfile is expected to be called from the root of this repo +## Maybe something like: docker build -t epicgames/jsstreamer:latest -f ./Extras/JSStreamer/Dockerfile . + WORKDIR /streamer -COPY /Extras/JSStreamer . +COPY /Common ./Common +COPY /Extras/JSStreamer ./Extras/JSStreamer +COPY ./package.json ./package.json +# Initiate NPM workspaces so we can install deps like our common lib using local built packages as opposed to remove published packages RUN npm install -CMD npm run develop +# Install and build common +RUN cd ./Common && npm install && npm run build + +# Install and build JSStream using the common lib we just build +RUN cd ./Extras/JSStreamer && npm install && npm run build + +# Run JSStreamer +CMD cd ./Extras/JSStreamer && npm run develop diff --git a/Extras/JSStreamer/package.json b/Extras/JSStreamer/package.json index ebf5ae24d..af96dfa25 100644 --- a/Extras/JSStreamer/package.json +++ b/Extras/JSStreamer/package.json @@ -35,8 +35,8 @@ "webpack-node-externals": "^3.0.0" }, "dependencies": { - "express": "^4.21.1", - "@epicgames-ps/lib-pixelstreamingcommon-ue5.5": "*" + "@epicgames-ps/lib-pixelstreamingcommon-ue5.5": "*", + "express": "^4.21.1" }, "repository": { "type": "git", diff --git a/package-lock.json b/package-lock.json index ffadd7845..df536e2e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -453,14 +453,6 @@ "node": ">=6.11.5" } }, - "Common/node_modules/minimist": { - "version": "1.2.8", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "Common/node_modules/neo-async": { "version": "2.6.2", "dev": true, @@ -843,7 +835,7 @@ "uuid": "^9.0.0" }, "devDependencies": { - "@playwright/test": "^1.49.0", + "@playwright/test": "^1.49.1", "@types/node": "^20.12.7", "@types/uuid": "^9.0.8" } @@ -1906,11 +1898,6 @@ "node": ">=4.0" } }, - "Extras/JSStreamer/node_modules/eventemitter3": { - "version": "4.0.7", - "dev": true, - "license": "MIT" - }, "Extras/JSStreamer/node_modules/events": { "version": "3.3.0", "dev": true, @@ -1952,25 +1939,6 @@ "flat": "cli.js" } }, - "Extras/JSStreamer/node_modules/follow-redirects": { - "version": "1.15.6", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "Extras/JSStreamer/node_modules/glob": { "version": "10.4.5", "dev": true, @@ -2050,14 +2018,6 @@ "node": ">=4" } }, - "Extras/JSStreamer/node_modules/he": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "bin": { - "he": "bin/he" - } - }, "Extras/JSStreamer/node_modules/hpack.js": { "version": "2.1.6", "dev": true, @@ -2190,19 +2150,6 @@ "dev": true, "license": "MIT" }, - "Extras/JSStreamer/node_modules/http-proxy": { - "version": "1.18.1", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.0", - "follow-redirects": "^1.0.0", - "requires-port": "^1.0.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, "Extras/JSStreamer/node_modules/http-proxy-middleware": { "version": "2.0.6", "dev": true, @@ -4532,11 +4479,6 @@ "node": ">=4.0" } }, - "Frontend/implementations/react/node_modules/eventemitter3": { - "version": "4.0.7", - "dev": true, - "license": "MIT" - }, "Frontend/implementations/react/node_modules/events": { "version": "3.3.0", "dev": true, @@ -4564,25 +4506,6 @@ "node": ">=0.8.0" } }, - "Frontend/implementations/react/node_modules/follow-redirects": { - "version": "1.15.6", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "Frontend/implementations/react/node_modules/fs-monkey": { "version": "1.0.3", "dev": true, @@ -4609,14 +4532,6 @@ "dev": true, "license": "MIT" }, - "Frontend/implementations/react/node_modules/he": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "bin": { - "he": "bin/he" - } - }, "Frontend/implementations/react/node_modules/hpack.js": { "version": "2.1.6", "dev": true, @@ -4796,19 +4711,6 @@ "dev": true, "license": "MIT" }, - "Frontend/implementations/react/node_modules/http-proxy": { - "version": "1.18.1", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.0", - "follow-redirects": "^1.0.0", - "requires-port": "^1.0.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, "Frontend/implementations/react/node_modules/http-proxy-middleware": { "version": "2.0.6", "dev": true, @@ -6851,11 +6753,6 @@ "node": ">=4.0" } }, - "Frontend/implementations/typescript/node_modules/eventemitter3": { - "version": "4.0.7", - "dev": true, - "license": "MIT" - }, "Frontend/implementations/typescript/node_modules/events": { "version": "3.3.0", "dev": true, @@ -6883,25 +6780,6 @@ "node": ">=0.8.0" } }, - "Frontend/implementations/typescript/node_modules/follow-redirects": { - "version": "1.15.6", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "Frontend/implementations/typescript/node_modules/fs-monkey": { "version": "1.0.3", "dev": true, @@ -6928,14 +6806,6 @@ "dev": true, "license": "MIT" }, - "Frontend/implementations/typescript/node_modules/he": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "bin": { - "he": "bin/he" - } - }, "Frontend/implementations/typescript/node_modules/hpack.js": { "version": "2.1.6", "dev": true, @@ -7115,19 +6985,6 @@ "dev": true, "license": "MIT" }, - "Frontend/implementations/typescript/node_modules/http-proxy": { - "version": "1.18.1", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.0", - "follow-redirects": "^1.0.0", - "requires-port": "^1.0.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, "Frontend/implementations/typescript/node_modules/http-proxy-middleware": { "version": "2.0.6", "dev": true, @@ -9844,12 +9701,12 @@ } }, "node_modules/@playwright/test": { - "version": "1.49.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.49.0.tgz", - "integrity": "sha512-DMulbwQURa8rNIQrf94+jPJQ4FmOVdpE5ZppRNvWVjvhC+6sOeo28r8MgIpQRYouXRtt/FCCXU7zn20jnHR4Qw==", + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.49.1.tgz", + "integrity": "sha512-Ky+BVzPz8pL6PQxHqNRW1k3mIyv933LML7HktS8uik0bUXNCdPhoS/kLihiO1tMf/egaJb4IutXd7UywvXEW+g==", "dev": true, "dependencies": { - "playwright": "1.49.0" + "playwright": "1.49.1" }, "bin": { "playwright": "cli.js" @@ -11748,6 +11605,12 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -12008,6 +11871,26 @@ "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", @@ -12375,6 +12258,15 @@ "node": ">= 0.4" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, "node_modules/html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -12408,6 +12300,20 @@ "node": ">= 0.8" } }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/http-proxy-agent": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", @@ -13766,6 +13672,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -13774,6 +13688,17 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -14180,12 +14105,12 @@ } }, "node_modules/playwright": { - "version": "1.49.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.0.tgz", - "integrity": "sha512-eKpmys0UFDnfNb3vfsf8Vx2LEOtflgRebl0Im2eQQnYMA4Aqd+Zw8bEOB+7ZKvN76901mRnqdsiOGKxzVTbi7A==", + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.1.tgz", + "integrity": "sha512-VYL8zLoNTBxVOrJBbDuRgDWa3i+mfQgDTrL8Ah9QXZ7ax4Dsj0MSq5bYgytRnDVVe+njoKnfsYkH3HzqVj5UZA==", "dev": true, "dependencies": { - "playwright-core": "1.49.0" + "playwright-core": "1.49.1" }, "bin": { "playwright": "cli.js" @@ -14198,9 +14123,9 @@ } }, "node_modules/playwright-core": { - "version": "1.49.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.0.tgz", - "integrity": "sha512-R+3KKTQF3npy5GTiKH/T+kdhoJfJojjHESR1YEWhYuEKRVfVaxH3+4+GvXE5xyCngCxhxnykk0Vlah9v8fs3jA==", + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.1.tgz", + "integrity": "sha512-BzmpVcs4kE2CH15rWfzpjzVGhWERJfmnXmniSyKeRZUs9Ws65m+RGIi7mjJK/euCegfn3i7jvqWeWyHe9y3Vgg==", "dev": true, "bin": { "playwright-core": "cli.js" @@ -15771,13 +15696,6 @@ "resolved": "SFU/mediasoup-sdp-bridge", "link": true }, - "SFU/node_modules/minimist": { - "version": "1.2.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "SFU/node_modules/netstring": { "version": "0.3.0", "engines": { @@ -16389,23 +16307,6 @@ "node": ">= 12.0.0" } }, - "Signalling/node_modules/minimist": { - "version": "1.2.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "Signalling/node_modules/mkdirp": { - "version": "0.5.6", - "license": "MIT", - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, "Signalling/node_modules/moment": { "version": "2.30.1", "license": "MIT", @@ -17583,23 +17484,6 @@ "version": "2.1.3", "license": "MIT" }, - "SignallingWebServer/node_modules/minimist": { - "version": "1.2.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "SignallingWebServer/node_modules/mkdirp": { - "version": "0.5.6", - "license": "MIT", - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, "SignallingWebServer/node_modules/moment": { "version": "2.30.1", "license": "MIT", From 911ead626a7be1d6c0524a4d251815de185f6842 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Mon, 20 Jan 2025 16:35:52 +1000 Subject: [PATCH 08/27] Added SDP offer/remote description set event to PixelStreaming API --- .../tests/peerconnection.spec.ts | 21 ++++++++++++++++++- .../PeerConnectionController.ts | 21 +++++++++++++++---- .../src/PixelStreaming/PixelStreaming.ts | 10 ++++++++- Frontend/library/src/Util/EventEmitter.ts | 20 +++++++++++++++++- .../WebRtcPlayer/WebRtcPlayerController.ts | 13 +++++++++--- 5 files changed, 75 insertions(+), 10 deletions(-) diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 4e892d301..0183a4acc 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -1,7 +1,7 @@ import { test } from './fixtures'; import { expect } from './matchers'; import * as helpers from './helpers'; -import { Flags, PixelStreaming, WebRtcSdpAnswerEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; +import { Flags, PixelStreaming, WebRtcSdpAnswerEvent, WebRtcSdpOfferEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; test('Test abs-capture-time header extension found for streamer', { tag: ['@capture-time'], @@ -20,15 +20,34 @@ test('Test abs-capture-time header extension found for streamer', { }); await page.goto(`/?StreamerId=${streamerId}`); + await page.waitForLoadState("load"); + + // Wait for the sdp offer + let getSdpOffer = new Promise(async (resolve) => { + + // Expose the resolve function to the browser context + await page.exposeFunction('resolveFromSdpOfferPromise', resolve); + + page.evaluate(() => { + window.pixelStreaming.addEventListener("webRtcSdpOffer", (e: WebRtcSdpOfferEvent) => { + resolveFromSdpOfferPromise(e.data.sdp); + }); + }); + + }); + await page.getByText('Click to start').click(); await helpers.waitForVideo(page); let localDescSdp: RTCSessionDescriptionInit = await localDescription; + let remoteDescSdp: RTCSessionDescriptionInit = await getSdpOffer; expect(localDescSdp.sdp).toBeDefined(); + expect(remoteDescSdp.sdp).toBeDefined(); // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer expect(localDescSdp.sdp).toContain("abs-capture-time"); + expect(remoteDescSdp.sdp).toContain("abs-capture-time"); }); test('Test abs-capture-time header extension found in PSInfra frontend', { diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 5a2c911fc..81496a9a7 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -89,12 +89,16 @@ export class PeerConnectionController { } /** - * + * Receive offer from UE side and process it as the remote description of this peer connection */ async receiveOffer(offer: RTCSessionDescriptionInit, config: Config) { Logger.Info('Receive Offer'); this.peerConnection?.setRemoteDescription(offer).then(() => { + + // Fire event for when remote offer description is set + this.onSetRemoteDescription(offer); + const isLocalhostConnection = location.hostname === 'localhost' || location.hostname === '127.0.0.1'; const isHttpsConnection = location.protocol === 'https:'; @@ -125,7 +129,7 @@ export class PeerConnectionController { return this.peerConnection?.setLocalDescription(Answer); }) .then(() => { - this.onSendWebRTCAnswer(this.peerConnection?.currentLocalDescription); + this.onSetLocalDescription(this.peerConnection?.currentLocalDescription); }) .catch((err) => { Logger.Error(`createAnswer() failed - ${err}`); @@ -603,11 +607,20 @@ export class PeerConnectionController { } /** - * Event to send the RTC Answer to the Signaling server + * Event fired when remote offer description is set. + * @param offer - RTC Offer + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + onSetRemoteDescription(offer: RTCSessionDescriptionInit) { + // Default Functionality: Do Nothing + } + + /** + * Event fire when local description answer is set. * @param answer - RTC Answer */ // eslint-disable-next-line @typescript-eslint/no-unused-vars - onSendWebRTCAnswer(answer: RTCSessionDescriptionInit) { + onSetLocalDescription(answer: RTCSessionDescriptionInit) { // Default Functionality: Do Nothing } diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index d806963fb..fb654087a 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -30,6 +30,7 @@ import { DataChannelLatencyTestResultEvent, PlayerCountEvent, WebRtcTCPRelayDetectedEvent, + WebRtcSdpOfferEvent, WebRtcSdpAnswerEvent } from '../Util/EventEmitter'; import { WebXRController } from '../WebXR/WebXRController'; @@ -464,7 +465,14 @@ export class PixelStreaming { } /** - * Set up functionality to happen after SDP has been generated and sent. + * Set up functionality to happen after offer has been set. + */ + _onWebRtcSdpOffer(offer: RTCSessionDescriptionInit) { + this._eventEmitter.dispatchEvent(new WebRtcSdpOfferEvent(offer)); + } + + /** + * Set up functionality to happen after SDP answer has set. */ _onWebRtcSdpAnswer(answer: RTCSessionDescriptionInit) { this._eventEmitter.dispatchEvent(new WebRtcSdpAnswerEvent(answer)); diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index e0945b0b6..c5f11f80c 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -91,7 +91,7 @@ export class WebRtcSdpEvent extends Event { } /** - * An event that is emitted after the SDP answer is generated and sent. + * An event that is emitted after the SDP answer is set. */ export class WebRtcSdpAnswerEvent extends Event { readonly type: 'webRtcSdpAnswer'; @@ -107,6 +107,23 @@ export class WebRtcSdpAnswerEvent extends Event { } } +/** + * An event that is emitted after the SDP offer is set. + */ +export class WebRtcSdpOfferEvent extends Event { + readonly type: 'webRtcSdpOffer'; + readonly data: { + /** The sdp offer */ + sdp: RTCSessionDescriptionInit; + }; + constructor(offer: RTCSessionDescriptionInit) { + super('webRtcSdpOffer'); + this.data = { + sdp: offer + }; + } +} + /** * An event that is emitted when auto connecting. */ @@ -566,6 +583,7 @@ export type PixelStreamingEvent = | AfkTimedOutEvent | VideoEncoderAvgQPEvent | WebRtcSdpEvent + | WebRtcSdpOfferEvent | WebRtcSdpAnswerEvent | WebRtcAutoConnectEvent | WebRtcConnectingEvent diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index 4c9a08742..93de862ba 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -1009,12 +1009,19 @@ export class WebRtcPlayerController { this.peerConnectionController.onVideoStats = (event: AggregatedStats) => this.handleVideoStats(event); /* When the Peer Connection wants to send an offer have it handled */ - this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => + this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => { this.handleSendWebRTCOffer(offer); + } - /* When the Peer Connection wants to send an answer have it handled */ - this.peerConnectionController.onSendWebRTCAnswer = (answer: RTCSessionDescriptionInit) => + /* Set event handler for when local answer description is set */ + this.peerConnectionController.onSetLocalDescription = (answer: RTCSessionDescriptionInit) => { this.handleSendWebRTCAnswer(answer); + } + + /* Set event handler for when remote offer description is set */ + this.peerConnectionController.onSetRemoteDescription = (offer: RTCSessionDescriptionInit) => { + this.pixelStreaming._onWebRtcSdpOffer(offer); + }; /* When the Peer Connection ice candidate is added have it handled */ this.peerConnectionController.onPeerIceCandidate = ( From 1f5b9f87d2303921909d63ad284d177159ab6ada Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 23 Jan 2025 16:07:58 +1000 Subject: [PATCH 09/27] Connect new latency calculation API to stats panel --- .../tests/peerconnection.spec.ts | 58 +++++- Frontend/library/src/Config/Config.ts | 2 +- .../LatencyCalculator.ts | 177 ++++++++++++++++++ .../PeerConnectionController.ts | 20 +- .../src/PixelStreaming/PixelStreaming.ts | 23 ++- Frontend/library/src/Util/EventEmitter.ts | 29 ++- .../WebRtcPlayer/WebRtcPlayerController.ts | 10 +- .../library/src/pixelstreamingfrontend.ts | 1 + .../ui-library/src/Application/Application.ts | 10 +- Frontend/ui-library/src/UI/LatencyTest.ts | 37 +++- Frontend/ui-library/src/UI/StatsPanel.ts | 68 ++++++- 11 files changed, 397 insertions(+), 38 deletions(-) create mode 100644 Frontend/library/src/PeerConnectionController/LatencyCalculator.ts diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 0183a4acc..c3fb2481a 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -1,7 +1,7 @@ import { test } from './fixtures'; import { expect } from './matchers'; import * as helpers from './helpers'; -import { Flags, PixelStreaming, WebRtcSdpAnswerEvent, WebRtcSdpOfferEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; +import { Flags, PixelStreaming, WebRtcSdpAnswerEvent, WebRtcSdpOfferEvent, LatencyCalculator, LatencyInfo, LatencyCalculatedEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; test('Test abs-capture-time header extension found for streamer', { tag: ['@capture-time'], @@ -89,3 +89,59 @@ test('Test abs-capture-time header extension found in PSInfra frontend', { // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer expect(answer.sdp).toContain("abs-capture-time"); }); + +test('Test latency calculation', { + tag: ['@capture-time'], +}, async ({ page, streamerPage, streamerId, browserName }) => { + + if(browserName !== 'chromium') { + // Chrome based browsers are the only ones that support. + test.skip(); + } + + await page.goto(`/?StreamerId=${streamerId}`); + + await page.waitForLoadState("load"); + + // Enable the flag for the capture extension + await page.evaluate(() => { + window.pixelStreaming.config.setFlagEnabled("EnableCaptureTimeExt", true); + }); + + await page.getByText('Click to start').click(); + await helpers.waitForVideo(page); + + // Wait for the latency info event to be fired + let latencyInfo: LatencyInfo = await page.evaluate(() => { + return new Promise((resolve) => { + window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => { + // Todo: Add event for `latencyCalculated` to Pixel Streaming API + }); + }); + }); + + expect(latencyInfo).toBeDefined(); + expect(latencyInfo.SenderLatencyMs).toBeDefined(); + expect(latencyInfo.AverageJitterBufferDelayMs).toBeDefined(); + expect(latencyInfo.AverageProcessingDelayMs).toBeDefined(); + expect(latencyInfo.RTTMs).toBeDefined(); + expect(latencyInfo.AverageAssemblyDelayMs).toBeDefined(); + expect(latencyInfo.AverageDecodeLatencyMs).toBeDefined(); + + // Sender side latency should be less than 60ms in pure CPU test + expect(latencyInfo.SenderLatencyMs).toBeLessThanOrEqual(60) + + // Expect jitter buffer/processing delay to be no greater than 3 frames @ 30fps + expect(latencyInfo.AverageJitterBufferDelayMs).toBeLessThanOrEqual(99); + expect(latencyInfo.AverageProcessingDelayMs).toBeLessThanOrEqual(99); + + // Expect RTT to be less than 10ms on loopback + expect(latencyInfo.RTTMs).toBeLessThanOrEqual(10); + + // Expect time to assemble frame from packets to be less than the frame rate itself at 30 fps + expect(latencyInfo.AverageAssemblyDelayMs).toBeLessThanOrEqual(33); + + // Expect CPU decoder to at least be able to do 30 fps decode + expect(latencyInfo.AverageDecodeLatencyMs).toBeLessThanOrEqual(33); + +}); diff --git a/Frontend/library/src/Config/Config.ts b/Frontend/library/src/Config/Config.ts index a44ce5d4a..5977b6ab5 100644 --- a/Frontend/library/src/Config/Config.ts +++ b/Frontend/library/src/Config/Config.ts @@ -557,7 +557,7 @@ export class Config { 'Enables the abs-capture-time RTP header extension', settings && Object.prototype.hasOwnProperty.call(settings, Flags.EnableCaptureTimeExt) ? settings[Flags.EnableCaptureTimeExt] - : false, + : true, useUrlParams ) ); diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts new file mode 100644 index 000000000..d3d203628 --- /dev/null +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -0,0 +1,177 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +import { Logger } from "@epicgames-ps/lib-pixelstreamingcommon-ue5.5"; +import { AggregatedStats } from "./AggregatedStats"; +import { CandidatePairStats } from "./CandidatePairStats"; + +/** + * Represents either a: + * - synchronization source: https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpReceiver/getSynchronizationSources + * - contributing source: https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpReceiver/getContributingSources + * Which also (if browser supports it) may optionall contain fields for captureTimestamp + senderCaptureTimeOffset + * if the abs-capture-time RTP header extension is enabled (currently this only works in Chromium based browsers). + */ +class RTCRtpCaptureSource { + timestamp: number; + captureTimestamp: number; + senderCaptureTimeOffset: number; +} + +/** + * Calculates a combination of latency statistics using purely WebRTC API. + */ +export class LatencyCalculator { + + public calculate(stats: AggregatedStats, receivers: RTCRtpReceiver[]) : LatencyInfo { + let latencyInfo = new LatencyInfo(); + + let activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); + + if(activeCandidatePair !== null && activeCandidatePair.currentRoundTripTime !== undefined && activeCandidatePair.currentRoundTripTime > 0) { + // Get RTT + latencyInfo.RTTMs = activeCandidatePair.currentRoundTripTime * 1000; + + // Calculate sender latency using the first valid video ssrc/csrc + let captureSource: RTCRtpCaptureSource = this.getCaptureSource(receivers); + if(captureSource !== null) { + latencyInfo.SenderLatencyMs = this.calculateSenderLatency(stats, captureSource); + } + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay + if(stats.inboundVideoStats.totalProcessingDelay !== undefined && stats.inboundVideoStats.framesDecoded !== undefined) { + latencyInfo.AverageProcessingDelayMs = (stats.inboundVideoStats.totalProcessingDelay / stats.inboundVideoStats.framesDecoded) * 1000; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferminimumdelay + if(stats.inboundVideoStats.jitterBufferDelay !== undefined && stats.inboundVideoStats.jitterBufferEmittedCount !== undefined) { + latencyInfo.AverageJitterBufferDelayMs = (stats.inboundVideoStats.jitterBufferDelay / stats.inboundVideoStats.jitterBufferEmittedCount) * 1000; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime + if(stats.inboundVideoStats.framesDecoded !== undefined && stats.inboundVideoStats.totalDecodeTime !== undefined) { + latencyInfo.AverageDecodeLatencyMs = (stats.inboundVideoStats.totalDecodeTime / stats.inboundVideoStats.framesDecoded) * 1000; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-framesassembledfrommultiplepackets + if(stats.inboundVideoStats.totalAssemblyTime !== undefined && stats.inboundVideoStats.framesAssembledFromMultiplePackets !== undefined) { + latencyInfo.AverageAssemblyDelayMs = (stats.inboundVideoStats.totalAssemblyTime / stats.inboundVideoStats.framesAssembledFromMultiplePackets) * 1000; + } + + // Calculate E2E latency as sender-side latency + network latency + receiver-side latency + if(latencyInfo.AverageProcessingDelayMs !== undefined && latencyInfo.AverageProcessingDelayMs > 0 && + latencyInfo.SenderLatencyMs != undefined && latencyInfo.SenderLatencyMs > 0 && + latencyInfo.RTTMs != undefined && latencyInfo.RTTMs > 0) { + latencyInfo.AverageE2ELatency = latencyInfo.SenderLatencyMs + (latencyInfo.RTTMs * 0.5) + latencyInfo.AverageProcessingDelayMs; + } + + return latencyInfo; + } + + private calculateSenderLatency(stats: AggregatedStats, captureSource: RTCRtpCaptureSource) : number { + + // The calculation performed in this function is as per the procedure defined here: + // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset + + // Get the sender capture in the sender's clock + let senderCaptureTimestamp = captureSource.captureTimestamp + captureSource.senderCaptureTimeOffset; + + let sendRecvClockOffset = this.calculateSenderReceiverClockOffset(stats); + + // This brings sender clock roughly inline with recv clock + let recvCaptureTimestampNTP = senderCaptureTimestamp + sendRecvClockOffset; + + // As defined in Chrome source: https://chromium.googlesource.com/external/webrtc/+/master/system_wrappers/include/clock.h#26 + const ntp1970 = 2208988800000; + + let recvCaptureTimestamp = recvCaptureTimestampNTP - ntp1970; + + let senderLatency = captureSource.timestamp - recvCaptureTimestamp; + + return senderLatency; + } + + /** + * Find the first valid ssrc or csrc that has capture time fields present from abs-capture-time header extension. + * @param receivers The RTP receviers this peer connection has. + * @returns A single valid ssrc or csrc that has capture time fields or null if there is none (e.g. in non-chromium browsers it will be null). + */ + private getCaptureSource(receivers: RTCRtpReceiver[]) : RTCRtpCaptureSource { + + // We only want video receivers + receivers = receivers.filter((receiver) => receiver.track.kind === "video"); + + for(let receiver of receivers) { + + // Go through all ssrc and csrc to check for capture timestamp + // Note: Conversion to `any` here is because TS does not have captureTimestamp etc defined in the types + // these fields only exist in Chromium currently. + let sources : any[] = receiver.getSynchronizationSources().concat(receiver.getContributingSources()); + + for(let src of sources) { + if(src.captureTimestamp !== undefined && src.senderCaptureTimeOffset !== undefined && src.timestamp !== undefined) { + let captureSrc = new RTCRtpCaptureSource(); + captureSrc.timestamp = src.timestamp; + captureSrc.captureTimestamp = src.captureTimestamp; + captureSrc.senderCaptureTimeOffset = src.senderCaptureTimeOffset; + return captureSrc; + } + } + } + + return null; + } + + private calculateSenderReceiverClockOffset(stats: AggregatedStats) { + + // The calculation performed in this function is as per the procedure defined here: + // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset + + let remoteVideoStatsArrivedTimestamp = stats.outBoundVideoStats.timestamp; + let remoteVideoStatsSentTimestamp = stats.outBoundVideoStats.remoteTimestamp; + + let activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); + let networkDelay = activeCandidatePair ? activeCandidatePair.currentRoundTripTime * 0.5 * 1000 : 0.0; + + if(remoteVideoStatsArrivedTimestamp !== undefined && remoteVideoStatsSentTimestamp !== undefined && networkDelay !== undefined) { + return remoteVideoStatsArrivedTimestamp - ( remoteVideoStatsSentTimestamp + networkDelay ); + } + + Logger.Warning("Could not get stats to calculate sender/receiver clock offset."); + return 0.0; + } + +} + +/** + * A collection of latency information calculated using the WebRTC API. + * Most stats are calculated following the spec: + * https://w3c.github.io/webrtc-stats/#dictionary-rtcinboundrtpstreamstats-members + */ +export class LatencyInfo { + + /** + * The time taken from sender frame capture to receiver frame receipt. + * Note: This can only be calculated if both offer and answer contain the + * the RTP header extension for `abs-capture-time`. + */ + public SenderLatencyMs: number | undefined = undefined; + + /* The round trip time (milliseconds) between each sender->receiver->sender */ + public RTTMs: number | undefined = undefined; + + /* Average time taken (milliseconds) from video packet receipt to post-decode. */ + public AverageProcessingDelayMs: number | undefined = undefined; + + /* Average time taken (milliseconds) inside the jitter buffer (which is post-receipt but pre-decode). */ + public AverageJitterBufferDelayMs: number | undefined = undefined; + + /* Average time taken (milliseconds) to decode a video frame. */ + public AverageDecodeLatencyMs: number | undefined = undefined; + + /* Average time taken (milliseconds) to between receipt of the first and last video packet of a. */ + public AverageAssemblyDelayMs: number | undefined = undefined; + + /* The sender latency + RTT/2 + processing delay */ + public AverageE2ELatency: number | undefined = undefined; +} \ No newline at end of file diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 81496a9a7..5975aaeef 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -7,6 +7,9 @@ import { parseRtpParameters, splitSections } from 'sdp'; import { RTCUtils } from '../Util/RTCUtils'; import { CodecStats } from './CodecStats'; import { SDPUtils } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; +import { LatencyCalculator, LatencyInfo } from './LatencyCalculator'; + +export const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; /** * Handles the Peer Connection @@ -19,6 +22,7 @@ export class PeerConnectionController { updateCodecSelection: boolean; videoTrack: MediaStreamTrack; audioTrack: MediaStreamTrack; + latencyCalculator: LatencyCalculator; /** * Create a new RTC Peer Connection client @@ -28,6 +32,7 @@ export class PeerConnectionController { constructor(options: RTCConfiguration, config: Config, preferredCodec: string) { this.config = config; this.createPeerConnection(options, preferredCodec); + this.latencyCalculator = new LatencyCalculator(); } createPeerConnection(options: RTCConfiguration, preferredCodec: string) { @@ -169,6 +174,11 @@ export class PeerConnectionController { Promise.allSettled([audioPromise, videoPromise]).then(() => { this.onVideoStats(this.aggregatedStats); + + // Calculate latency using stats and video receivers and then call the handling function + let latencyInfo: LatencyInfo = this.latencyCalculator.calculate(this.aggregatedStats, this.peerConnection.getReceivers()); + this.onLatencyCalculated(latencyInfo); + // Update the preferred codec selection based on what was actually negotiated if (this.updateCodecSelection && !!this.aggregatedStats.inboundVideoStats.codecId) { // Construct the qualified codec name from the mimetype and fmtp @@ -244,7 +254,6 @@ export class PeerConnectionController { // Add abs-capture-time RTP header extension if we have enabled the setting if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt)) { - const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; mungedSDP = SDPUtils.addHeaderExtensionToSdp(mungedSDP, kAbsCaptureTime); } @@ -597,6 +606,15 @@ export class PeerConnectionController { // Default Functionality: Do Nothing } + /** + * And override event for when latency info is calculated + * @param latencyInfo - Calculated latency information. + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + onLatencyCalculated(latencyInfo: LatencyInfo) { + // Default Functionality: Do Nothing + } + /** * Event to send the RTC offer to the Signaling server * @param offer - RTC Offer diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index fb654087a..ea5298369 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -11,6 +11,7 @@ import { OnScreenKeyboard } from '../UI/OnScreenKeyboard'; import { PixelStreamingEventEmitter, InitialSettingsEvent, + LatencyCalculatedEvent, LatencyTestResultEvent, PixelStreamingEvent, StatsReceivedEvent, @@ -45,6 +46,7 @@ import { } from '../DataChannel/DataChannelLatencyTestResults'; import { RTCUtils } from '../Util/RTCUtils'; import { IURLSearchParams } from '../Util/IURLSearchParams'; +import { LatencyInfo } from '../PeerConnectionController/LatencyCalculator'; export interface PixelStreamingOverrides { /** The DOM element where Pixel Streaming video and user input event handlers are attached to. @@ -451,35 +453,42 @@ export class PixelStreaming { } /** - * Emit an event on auto connecting + * Internal function to emit an event when auto connecting occurs */ _onWebRtcAutoConnect() { this._eventEmitter.dispatchEvent(new WebRtcAutoConnectEvent()); } /** - * Set up functionality to happen when SDP negotiation is fully finished. + * Internal function to emit an event for when SDP negotiation is fully finished. */ _onWebRtcSdp() { this._eventEmitter.dispatchEvent(new WebRtcSdpEvent()); } /** - * Set up functionality to happen after offer has been set. + * Internal function to emit an SDP offer after it has been set. */ _onWebRtcSdpOffer(offer: RTCSessionDescriptionInit) { - this._eventEmitter.dispatchEvent(new WebRtcSdpOfferEvent(offer)); + this._eventEmitter.dispatchEvent(new WebRtcSdpOfferEvent({ sdp: offer })); } /** - * Set up functionality to happen after SDP answer has set. + * Internal function to emit an SDP answer after it has been set. */ _onWebRtcSdpAnswer(answer: RTCSessionDescriptionInit) { - this._eventEmitter.dispatchEvent(new WebRtcSdpAnswerEvent(answer)); + this._eventEmitter.dispatchEvent(new WebRtcSdpAnswerEvent({ sdp: answer })); } /** - * Emits a StreamLoading event + * Internal function call to emit a `latencyCalculated` event. + */ + _onLatencyCalculated(latencyInfo: LatencyInfo) { + this._eventEmitter.dispatchEvent(new LatencyCalculatedEvent({ latencyInfo })); + } + + /** + * Internal function to emits a StreamLoading event */ _onStreamLoading() { this._eventEmitter.dispatchEvent(new StreamLoadingEvent()); diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index c5f11f80c..b64386100 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -1,7 +1,7 @@ import { FlagsIds, NumericParametersIds, OptionParametersIds, TextParametersIds } from '../Config/Config'; import { LatencyTestResults } from '../DataChannel/LatencyTestResults'; import { AggregatedStats } from '../PeerConnectionController/AggregatedStats'; -import { InitialSettings } from '../pixelstreamingfrontend'; +import { InitialSettings, LatencyInfo } from '../pixelstreamingfrontend'; import { Messages } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; import { SettingFlag } from '../Config/SettingFlag'; import { SettingNumber } from '../Config/SettingNumber'; @@ -99,11 +99,9 @@ export class WebRtcSdpAnswerEvent extends Event { /** The sdp answer */ sdp: RTCSessionDescriptionInit; }; - constructor(answer: RTCSessionDescriptionInit) { + constructor(data: WebRtcSdpAnswerEvent["data"]) { super('webRtcSdpAnswer'); - this.data = { - sdp: answer - }; + this.data = data; } } @@ -116,11 +114,9 @@ export class WebRtcSdpOfferEvent extends Event { /** The sdp offer */ sdp: RTCSessionDescriptionInit; }; - constructor(offer: RTCSessionDescriptionInit) { + constructor(data: WebRtcSdpOfferEvent["data"]) { super('webRtcSdpOffer'); - this.data = { - sdp: offer - }; + this.data = data; } } @@ -416,6 +412,20 @@ export class LatencyTestResultEvent extends Event { } } +/** + * An event that is emitted everytime latency is calculated using the WebRTC stats API. + */ +export class LatencyCalculatedEvent extends Event { + readonly type: 'latencyCalculated'; + readonly data: { + latencyInfo: LatencyInfo; + } + constructor(data: LatencyCalculatedEvent['data']) { + super('latencyCalculated'); + this.data = data; + } +} + /** * An event that is emitted when receiving data channel latency test response from server. * This event is handled by DataChannelLatencyTestController @@ -606,6 +616,7 @@ export type PixelStreamingEvent = | StatsReceivedEvent | StreamerListMessageEvent | StreamerIDChangedMessageEvent + | LatencyCalculatedEvent | LatencyTestResultEvent | DataChannelLatencyTestResponseEvent | DataChannelLatencyTestResultEvent diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index 93de862ba..54b3db87a 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -54,6 +54,7 @@ import { import { IURLSearchParams } from '../Util/IURLSearchParams'; import { IInputController } from '../Inputs/IInputController'; import { GamepadController } from '../Inputs/GamepadController'; +import { LatencyInfo } from '../PeerConnectionController/LatencyCalculator'; /** * Entry point for the WebRTC Player @@ -1006,7 +1007,14 @@ export class WebRtcPlayerController { ); // set up peer connection controller video stats - this.peerConnectionController.onVideoStats = (event: AggregatedStats) => this.handleVideoStats(event); + this.peerConnectionController.onVideoStats = (event: AggregatedStats) => { + this.handleVideoStats(event); + } + + /* Set event handler for latency information is calculated, handle the event by propogating to the PixelStreaming API */ + this.peerConnectionController.onLatencyCalculated = (latencyInfo: LatencyInfo) => { + this.pixelStreaming._onLatencyCalculated(latencyInfo); + } /* When the Peer Connection wants to send an offer have it handled */ this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => { diff --git a/Frontend/library/src/pixelstreamingfrontend.ts b/Frontend/library/src/pixelstreamingfrontend.ts index 21fe5ad39..e93aade3b 100644 --- a/Frontend/library/src/pixelstreamingfrontend.ts +++ b/Frontend/library/src/pixelstreamingfrontend.ts @@ -43,6 +43,7 @@ export { CandidateStat } from './PeerConnectionController/CandidateStat'; export { DataChannelStats } from './PeerConnectionController/DataChannelStats'; export { InboundAudioStats, InboundVideoStats } from './PeerConnectionController/InboundRTPStats'; export { OutBoundVideoStats } from './PeerConnectionController/OutBoundRTPStats'; +export * from "./PeerConnectionController/LatencyCalculator" export * from './DataChannel/DataChannelLatencyTestResults'; export * from './Util/EventEmitter'; export * from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; diff --git a/Frontend/ui-library/src/Application/Application.ts b/Frontend/ui-library/src/Application/Application.ts index 8acaf486a..100f0caf5 100644 --- a/Frontend/ui-library/src/Application/Application.ts +++ b/Frontend/ui-library/src/Application/Application.ts @@ -8,7 +8,8 @@ import { LatencyTestResults, InitialSettings, Messages, - DataChannelLatencyTestResult + DataChannelLatencyTestResult, + LatencyInfo } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; import { OverlayBase } from '../Overlay/BaseOverlay'; import { ActionOverlay } from '../Overlay/ActionOverlay'; @@ -331,6 +332,9 @@ export class Application { this.stream.addEventListener('statsReceived', ({ data: { aggregatedStats } }) => this.onStatsReceived(aggregatedStats) ); + this.stream.addEventListener('latencyCalculated', ({ data: { latencyInfo } }) => + this.onLatencyUpdate(latencyInfo) + ); this.stream.addEventListener('latencyTestResult', ({ data: { latencyTimings } }) => this.onLatencyTestResults(latencyTimings) ); @@ -623,6 +627,10 @@ export class Application { this.statsPanel?.handleStats(aggregatedStats); } + onLatencyUpdate(latencyInfo: LatencyInfo) { + this.statsPanel?.handleLatencyInfo(latencyInfo); + } + onLatencyTestResults(latencyTimings: LatencyTestResults) { this.statsPanel?.latencyTest.handleTestResult(latencyTimings); } diff --git a/Frontend/ui-library/src/UI/LatencyTest.ts b/Frontend/ui-library/src/UI/LatencyTest.ts index 07f3b2e93..197850706 100644 --- a/Frontend/ui-library/src/UI/LatencyTest.ts +++ b/Frontend/ui-library/src/UI/LatencyTest.ts @@ -70,22 +70,43 @@ export class LatencyTest { public handleTestResult(latencyTimings: LatencyTestResults) { Logger.Info(JSON.stringify(latencyTimings)); let latencyStatsInnerHTML = ''; - latencyStatsInnerHTML += '
Net latency RTT (ms): ' + latencyTimings.networkLatency + '
'; - latencyStatsInnerHTML += '
UE Encode (ms): ' + latencyTimings.EncodeMs + '
'; - latencyStatsInnerHTML += '
UE Capture (ms): ' + latencyTimings.CaptureToSendMs + '
'; - latencyStatsInnerHTML += - '
Browser send latency (ms): ' + latencyTimings.browserSendLatency + '
'; - latencyStatsInnerHTML += + + if(latencyTimings.networkLatency !== undefined && latencyTimings.networkLatency > 0) { + latencyStatsInnerHTML += '
Net latency RTT (ms): ' + latencyTimings.networkLatency + '
'; + } + + if(latencyTimings.EncodeMs !== undefined && latencyTimings.EncodeMs > 0) { + latencyStatsInnerHTML += '
UE Encode (ms): ' + latencyTimings.EncodeMs + '
'; + } + + if(latencyTimings.CaptureToSendMs !== undefined && latencyTimings.CaptureToSendMs > 0) { + latencyStatsInnerHTML += '
UE Capture (ms): ' + latencyTimings.CaptureToSendMs + '
'; + } + + if(latencyTimings.browserSendLatency !== undefined && latencyTimings.browserSendLatency > 0) { + latencyStatsInnerHTML += '
Browser send latency (ms): ' + latencyTimings.browserSendLatency + '
'; + } + + if(latencyTimings.frameDisplayDeltaTimeMs !== undefined && latencyTimings.browserReceiptTimeMs !== undefined) { + latencyStatsInnerHTML += latencyTimings.frameDisplayDeltaTimeMs && latencyTimings.browserReceiptTimeMs ? '
Browser receive latency (ms): ' + latencyTimings.frameDisplayDeltaTimeMs + '
' : ''; - latencyStatsInnerHTML += + } + + if(latencyTimings.latencyExcludingDecode !== undefined) { + latencyStatsInnerHTML += '
Total latency (excluding browser) (ms): ' + latencyTimings.latencyExcludingDecode + '
'; - latencyStatsInnerHTML += latencyTimings.endToEndLatency + } + + if(latencyTimings.endToEndLatency !== undefined) { + latencyStatsInnerHTML += latencyTimings.endToEndLatency ? '
Total latency (ms): ' + latencyTimings.endToEndLatency + '
' : ''; + } + this.latencyTestResultsElement.innerHTML = latencyStatsInnerHTML; } } diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index d874ae1e3..5199f9026 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -3,6 +3,7 @@ import { LatencyTest } from './LatencyTest'; import { CandidatePairStats, + LatencyInfo, Logger, PixelStreaming, PixelStreamingSettings @@ -297,7 +298,7 @@ export class StatsPanel { const netRTT = Object.prototype.hasOwnProperty.call(activeCandidatePair, 'currentRoundTripTime') && stats.isNumber(activeCandidatePair.currentRoundTripTime) - ? numberFormat.format(activeCandidatePair.currentRoundTripTime * 1000) + ? Math.ceil(activeCandidatePair.currentRoundTripTime * 1000).toString() : "Can't calculate"; this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT); @@ -310,18 +311,67 @@ export class StatsPanel { ); // QP - this.addOrUpdateStat( - 'QPStat', - 'Video quantization parameter', - stats.sessionStats.videoEncoderAvgQP.toString() - ); - - // todo: - //statsText += `
Browser receive to composite (ms): ${stats.inboundVideoStats.receiveToCompositeMs}
`; + if(stats.sessionStats.videoEncoderAvgQP !== undefined && !Number.isNaN(stats.sessionStats.videoEncoderAvgQP)) { + this.addOrUpdateStat( + 'QPStat', + 'Video quantization parameter', + stats.sessionStats.videoEncoderAvgQP.toString() + ); + } Logger.Info(`--------- Stats ---------\n ${JSON.stringify(stats)}\n------------------------`); } + public handleLatencyInfo(latencyInfo: LatencyInfo) { + if(latencyInfo.SenderLatencyMs !== undefined && latencyInfo.SenderLatencyMs > 0) { + this.addOrUpdateStat( + 'SenderSideLatency', + 'Sender latency (ms)', + Math.ceil(latencyInfo.SenderLatencyMs).toString() + ); + } + + if(latencyInfo.AverageAssemblyDelayMs !== undefined && latencyInfo.AverageAssemblyDelayMs > 0) { + this.addOrUpdateStat( + 'AvgAssemblyDelay', + 'Assembly delay (ms)', + Math.ceil(latencyInfo.AverageAssemblyDelayMs).toString() + ); + } + + if(latencyInfo.AverageDecodeLatencyMs !== undefined && latencyInfo.AverageDecodeLatencyMs > 0) { + this.addOrUpdateStat( + 'AvgDecodeDelay', + 'Decode time (ms)', + Math.ceil(latencyInfo.AverageDecodeLatencyMs).toString() + ); + } + + if(latencyInfo.AverageJitterBufferDelayMs !== undefined && latencyInfo.AverageJitterBufferDelayMs > 0) { + this.addOrUpdateStat( + 'AvgJitterBufferDelay', + 'Jitter buffer (ms)', + Math.ceil(latencyInfo.AverageJitterBufferDelayMs).toString() + ); + } + + if(latencyInfo.AverageProcessingDelayMs !== undefined && latencyInfo.AverageProcessingDelayMs > 0) { + this.addOrUpdateStat( + 'AvgProcessingDelay', + 'Processing delay (ms)', + Math.ceil(latencyInfo.AverageProcessingDelayMs).toString() + ); + } + + if(latencyInfo.AverageE2ELatency !== undefined && latencyInfo.AverageE2ELatency > 0) { + this.addOrUpdateStat( + 'AvgE2ELatency', + 'Total latency (ms)', + Math.ceil(latencyInfo.AverageE2ELatency).toString() + ); + } + } + /** * Adds a new stat to the stats results in the DOM or updates an exiting stat. * @param id - The id of the stat to add/update. From e15961a0a257e03043e9a1f8b271ab777cb0fab7 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 24 Jan 2025 14:00:03 +1000 Subject: [PATCH 10/27] Added latency info to tests, fixed linting --- Extras/FrontendTests/tests/fixtures.ts | 8 + .../tests/peerconnection.spec.ts | 14 +- Extras/JSStreamer/src/streamer.ts | 7 + .../LatencyCalculator.ts | 327 ++++++++++-------- .../PeerConnectionController.ts | 6 +- Frontend/library/src/Util/EventEmitter.ts | 6 +- .../WebRtcPlayer/WebRtcPlayerController.ts | 8 +- .../library/src/pixelstreamingfrontend.ts | 2 +- Frontend/ui-library/src/UI/LatencyTest.ts | 38 +- Frontend/ui-library/src/UI/StatsPanel.ts | 20 +- 10 files changed, 251 insertions(+), 185 deletions(-) diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index 37986ca6d..7a0b73c5f 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -11,6 +11,11 @@ export const test = base.extend({ const streamerPage = await context.newPage(); await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}` + `${process.env.STREAMER_SIGNALLING_URL !== undefined ? '?SignallingURL=' + process.env.STREAMER_SIGNALLING_URL : ""}`); await use(streamerPage); + + // this is called after test is run by `use` + streamerPage.evaluate(() => { + window.streamer.stopStreaming(); + }); }, streamerId: async ({ streamerPage }, use) => { @@ -36,3 +41,6 @@ export const test = base.extend({ } }); + +// Ensure tests run in serial as we don't want a clash with multiple peers putting things in different states +test.describe.configure({ mode: 'serial' }); \ No newline at end of file diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index c3fb2481a..a01d8d8d1 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -115,7 +115,9 @@ test('Test latency calculation', { let latencyInfo: LatencyInfo = await page.evaluate(() => { return new Promise((resolve) => { window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => { - // Todo: Add event for `latencyCalculated` to Pixel Streaming API + if(e.data.latencyInfo && e.data.latencyInfo.SenderLatencyMs) { + resolve(e.data.latencyInfo); + } }); }); }); @@ -128,12 +130,12 @@ test('Test latency calculation', { expect(latencyInfo.AverageAssemblyDelayMs).toBeDefined(); expect(latencyInfo.AverageDecodeLatencyMs).toBeDefined(); - // Sender side latency should be less than 60ms in pure CPU test - expect(latencyInfo.SenderLatencyMs).toBeLessThanOrEqual(60) + // Sender side latency should be less than 500ms in pure CPU test + expect(latencyInfo.SenderLatencyMs).toBeLessThanOrEqual(500) - // Expect jitter buffer/processing delay to be no greater than 3 frames @ 30fps - expect(latencyInfo.AverageJitterBufferDelayMs).toBeLessThanOrEqual(99); - expect(latencyInfo.AverageProcessingDelayMs).toBeLessThanOrEqual(99); + // Expect jitter buffer/processing delay to be no greater than 500ms on local link + expect(latencyInfo.AverageJitterBufferDelayMs).toBeLessThanOrEqual(500); + expect(latencyInfo.AverageProcessingDelayMs).toBeLessThanOrEqual(500); // Expect RTT to be less than 10ms on loopback expect(latencyInfo.RTTMs).toBeLessThanOrEqual(10); diff --git a/Extras/JSStreamer/src/streamer.ts b/Extras/JSStreamer/src/streamer.ts index 8b79bf34a..c598b021d 100644 --- a/Extras/JSStreamer/src/streamer.ts +++ b/Extras/JSStreamer/src/streamer.ts @@ -129,6 +129,13 @@ export class Streamer extends EventEmitter { this.transport.connect(signallingURL); } + stopStreaming() { + this.transport.disconnect(1000, "Normal shutdown by calling stopStreaming"); + for(let peer of this.playerMap.values()) { + peer.peerConnection.close(); + } + } + handleConfigMessage(msg: Messages.config) { if(msg.peerConnectionOptions !== undefined) { this.peerConnectionOptions = msg.peerConnectionOptions; diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts index d3d203628..67066d77d 100644 --- a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -1,8 +1,8 @@ // Copyright Epic Games, Inc. All Rights Reserved. -import { Logger } from "@epicgames-ps/lib-pixelstreamingcommon-ue5.5"; -import { AggregatedStats } from "./AggregatedStats"; -import { CandidatePairStats } from "./CandidatePairStats"; +import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; +import { AggregatedStats } from './AggregatedStats'; +import { CandidatePairStats } from './CandidatePairStats'; /** * Represents either a: @@ -12,135 +12,171 @@ import { CandidatePairStats } from "./CandidatePairStats"; * if the abs-capture-time RTP header extension is enabled (currently this only works in Chromium based browsers). */ class RTCRtpCaptureSource { - timestamp: number; - captureTimestamp: number; - senderCaptureTimeOffset: number; + timestamp: number; + captureTimestamp: number; + senderCaptureTimeOffset: number; } /** * Calculates a combination of latency statistics using purely WebRTC API. */ export class LatencyCalculator { - - public calculate(stats: AggregatedStats, receivers: RTCRtpReceiver[]) : LatencyInfo { - let latencyInfo = new LatencyInfo(); - - let activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); - - if(activeCandidatePair !== null && activeCandidatePair.currentRoundTripTime !== undefined && activeCandidatePair.currentRoundTripTime > 0) { - // Get RTT - latencyInfo.RTTMs = activeCandidatePair.currentRoundTripTime * 1000; - - // Calculate sender latency using the first valid video ssrc/csrc - let captureSource: RTCRtpCaptureSource = this.getCaptureSource(receivers); - if(captureSource !== null) { - latencyInfo.SenderLatencyMs = this.calculateSenderLatency(stats, captureSource); - } - } - - // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay - if(stats.inboundVideoStats.totalProcessingDelay !== undefined && stats.inboundVideoStats.framesDecoded !== undefined) { - latencyInfo.AverageProcessingDelayMs = (stats.inboundVideoStats.totalProcessingDelay / stats.inboundVideoStats.framesDecoded) * 1000; - } - - // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferminimumdelay - if(stats.inboundVideoStats.jitterBufferDelay !== undefined && stats.inboundVideoStats.jitterBufferEmittedCount !== undefined) { - latencyInfo.AverageJitterBufferDelayMs = (stats.inboundVideoStats.jitterBufferDelay / stats.inboundVideoStats.jitterBufferEmittedCount) * 1000; - } - - // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime - if(stats.inboundVideoStats.framesDecoded !== undefined && stats.inboundVideoStats.totalDecodeTime !== undefined) { - latencyInfo.AverageDecodeLatencyMs = (stats.inboundVideoStats.totalDecodeTime / stats.inboundVideoStats.framesDecoded) * 1000; - } - - // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-framesassembledfrommultiplepackets - if(stats.inboundVideoStats.totalAssemblyTime !== undefined && stats.inboundVideoStats.framesAssembledFromMultiplePackets !== undefined) { - latencyInfo.AverageAssemblyDelayMs = (stats.inboundVideoStats.totalAssemblyTime / stats.inboundVideoStats.framesAssembledFromMultiplePackets) * 1000; - } - - // Calculate E2E latency as sender-side latency + network latency + receiver-side latency - if(latencyInfo.AverageProcessingDelayMs !== undefined && latencyInfo.AverageProcessingDelayMs > 0 && - latencyInfo.SenderLatencyMs != undefined && latencyInfo.SenderLatencyMs > 0 && - latencyInfo.RTTMs != undefined && latencyInfo.RTTMs > 0) { - latencyInfo.AverageE2ELatency = latencyInfo.SenderLatencyMs + (latencyInfo.RTTMs * 0.5) + latencyInfo.AverageProcessingDelayMs; - } - - return latencyInfo; - } - - private calculateSenderLatency(stats: AggregatedStats, captureSource: RTCRtpCaptureSource) : number { - - // The calculation performed in this function is as per the procedure defined here: - // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset - - // Get the sender capture in the sender's clock - let senderCaptureTimestamp = captureSource.captureTimestamp + captureSource.senderCaptureTimeOffset; - - let sendRecvClockOffset = this.calculateSenderReceiverClockOffset(stats); - - // This brings sender clock roughly inline with recv clock - let recvCaptureTimestampNTP = senderCaptureTimestamp + sendRecvClockOffset; - - // As defined in Chrome source: https://chromium.googlesource.com/external/webrtc/+/master/system_wrappers/include/clock.h#26 - const ntp1970 = 2208988800000; - - let recvCaptureTimestamp = recvCaptureTimestampNTP - ntp1970; - - let senderLatency = captureSource.timestamp - recvCaptureTimestamp; - - return senderLatency; - } - - /** - * Find the first valid ssrc or csrc that has capture time fields present from abs-capture-time header extension. - * @param receivers The RTP receviers this peer connection has. - * @returns A single valid ssrc or csrc that has capture time fields or null if there is none (e.g. in non-chromium browsers it will be null). - */ - private getCaptureSource(receivers: RTCRtpReceiver[]) : RTCRtpCaptureSource { - - // We only want video receivers - receivers = receivers.filter((receiver) => receiver.track.kind === "video"); - - for(let receiver of receivers) { - - // Go through all ssrc and csrc to check for capture timestamp - // Note: Conversion to `any` here is because TS does not have captureTimestamp etc defined in the types - // these fields only exist in Chromium currently. - let sources : any[] = receiver.getSynchronizationSources().concat(receiver.getContributingSources()); - - for(let src of sources) { - if(src.captureTimestamp !== undefined && src.senderCaptureTimeOffset !== undefined && src.timestamp !== undefined) { - let captureSrc = new RTCRtpCaptureSource(); - captureSrc.timestamp = src.timestamp; - captureSrc.captureTimestamp = src.captureTimestamp; - captureSrc.senderCaptureTimeOffset = src.senderCaptureTimeOffset; - return captureSrc; - } - } - } - - return null; - } - - private calculateSenderReceiverClockOffset(stats: AggregatedStats) { - - // The calculation performed in this function is as per the procedure defined here: - // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset - - let remoteVideoStatsArrivedTimestamp = stats.outBoundVideoStats.timestamp; - let remoteVideoStatsSentTimestamp = stats.outBoundVideoStats.remoteTimestamp; - - let activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); - let networkDelay = activeCandidatePair ? activeCandidatePair.currentRoundTripTime * 0.5 * 1000 : 0.0; - - if(remoteVideoStatsArrivedTimestamp !== undefined && remoteVideoStatsSentTimestamp !== undefined && networkDelay !== undefined) { - return remoteVideoStatsArrivedTimestamp - ( remoteVideoStatsSentTimestamp + networkDelay ); - } - - Logger.Warning("Could not get stats to calculate sender/receiver clock offset."); - return 0.0; - } - + public calculate(stats: AggregatedStats, receivers: RTCRtpReceiver[]): LatencyInfo { + const latencyInfo = new LatencyInfo(); + + const activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); + + if ( + activeCandidatePair !== null && + activeCandidatePair.currentRoundTripTime !== undefined && + activeCandidatePair.currentRoundTripTime > 0 + ) { + // Get RTT + latencyInfo.RTTMs = activeCandidatePair.currentRoundTripTime * 1000; + + // Calculate sender latency using the first valid video ssrc/csrc + const captureSource: RTCRtpCaptureSource = this.getCaptureSource(receivers); + if (captureSource !== null) { + latencyInfo.SenderLatencyMs = this.calculateSenderLatency(stats, captureSource); + } + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay + if ( + stats.inboundVideoStats.totalProcessingDelay !== undefined && + stats.inboundVideoStats.framesDecoded !== undefined + ) { + latencyInfo.AverageProcessingDelayMs = + (stats.inboundVideoStats.totalProcessingDelay / stats.inboundVideoStats.framesDecoded) * 1000; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferminimumdelay + if ( + stats.inboundVideoStats.jitterBufferDelay !== undefined && + stats.inboundVideoStats.jitterBufferEmittedCount !== undefined + ) { + latencyInfo.AverageJitterBufferDelayMs = + (stats.inboundVideoStats.jitterBufferDelay / + stats.inboundVideoStats.jitterBufferEmittedCount) * + 1000; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime + if ( + stats.inboundVideoStats.framesDecoded !== undefined && + stats.inboundVideoStats.totalDecodeTime !== undefined + ) { + latencyInfo.AverageDecodeLatencyMs = + (stats.inboundVideoStats.totalDecodeTime / stats.inboundVideoStats.framesDecoded) * 1000; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-framesassembledfrommultiplepackets + if ( + stats.inboundVideoStats.totalAssemblyTime !== undefined && + stats.inboundVideoStats.framesAssembledFromMultiplePackets !== undefined + ) { + latencyInfo.AverageAssemblyDelayMs = + (stats.inboundVideoStats.totalAssemblyTime / + stats.inboundVideoStats.framesAssembledFromMultiplePackets) * + 1000; + } + + // Calculate E2E latency as sender-side latency + network latency + receiver-side latency + if ( + latencyInfo.AverageProcessingDelayMs !== undefined && + latencyInfo.AverageProcessingDelayMs > 0 && + latencyInfo.SenderLatencyMs != undefined && + latencyInfo.SenderLatencyMs > 0 && + latencyInfo.RTTMs != undefined && + latencyInfo.RTTMs > 0 + ) { + latencyInfo.AverageE2ELatency = + latencyInfo.SenderLatencyMs + latencyInfo.RTTMs * 0.5 + latencyInfo.AverageProcessingDelayMs; + } + + return latencyInfo; + } + + private calculateSenderLatency(stats: AggregatedStats, captureSource: RTCRtpCaptureSource): number { + // The calculation performed in this function is as per the procedure defined here: + // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset + + // Get the sender capture in the sender's clock + const senderCaptureTimestamp = captureSource.captureTimestamp + captureSource.senderCaptureTimeOffset; + + const sendRecvClockOffset = this.calculateSenderReceiverClockOffset(stats); + + // This brings sender clock roughly inline with recv clock + const recvCaptureTimestampNTP = senderCaptureTimestamp + sendRecvClockOffset; + + // As defined in Chrome source: https://chromium.googlesource.com/external/webrtc/+/master/system_wrappers/include/clock.h#26 + const ntp1970 = 2208988800000; + + const recvCaptureTimestamp = recvCaptureTimestampNTP - ntp1970; + + const senderLatency = captureSource.timestamp - recvCaptureTimestamp; + + return senderLatency; + } + + /** + * Find the first valid ssrc or csrc that has capture time fields present from abs-capture-time header extension. + * @param receivers The RTP receviers this peer connection has. + * @returns A single valid ssrc or csrc that has capture time fields or null if there is none (e.g. in non-chromium browsers it will be null). + */ + private getCaptureSource(receivers: RTCRtpReceiver[]): RTCRtpCaptureSource { + // We only want video receivers + receivers = receivers.filter((receiver) => receiver.track.kind === 'video'); + + for (const receiver of receivers) { + // Go through all ssrc and csrc to check for capture timestamp + // Note: Conversion to `any` here is because TS does not have captureTimestamp etc defined in the types + // these fields only exist in Chromium currently. + const sources: any[] = receiver + .getSynchronizationSources() + .concat(receiver.getContributingSources()); + + for (const src of sources) { + if ( + src.captureTimestamp !== undefined && + src.senderCaptureTimeOffset !== undefined && + src.timestamp !== undefined + ) { + const captureSrc = new RTCRtpCaptureSource(); + captureSrc.timestamp = src.timestamp; + captureSrc.captureTimestamp = src.captureTimestamp; + captureSrc.senderCaptureTimeOffset = src.senderCaptureTimeOffset; + return captureSrc; + } + } + } + + return null; + } + + private calculateSenderReceiverClockOffset(stats: AggregatedStats) { + // The calculation performed in this function is as per the procedure defined here: + // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset + + const remoteVideoStatsArrivedTimestamp = stats.outBoundVideoStats.timestamp; + const remoteVideoStatsSentTimestamp = stats.outBoundVideoStats.remoteTimestamp; + + const activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); + const networkDelay = activeCandidatePair + ? activeCandidatePair.currentRoundTripTime * 0.5 * 1000 + : 0.0; + + if ( + remoteVideoStatsArrivedTimestamp !== undefined && + remoteVideoStatsSentTimestamp !== undefined && + networkDelay !== undefined + ) { + return remoteVideoStatsArrivedTimestamp - (remoteVideoStatsSentTimestamp + networkDelay); + } + + Logger.Warning('Could not get stats to calculate sender/receiver clock offset.'); + return 0.0; + } } /** @@ -149,29 +185,28 @@ export class LatencyCalculator { * https://w3c.github.io/webrtc-stats/#dictionary-rtcinboundrtpstreamstats-members */ export class LatencyInfo { + /** + * The time taken from sender frame capture to receiver frame receipt. + * Note: This can only be calculated if both offer and answer contain the + * the RTP header extension for `abs-capture-time`. + */ + public SenderLatencyMs: number | undefined = undefined; - /** - * The time taken from sender frame capture to receiver frame receipt. - * Note: This can only be calculated if both offer and answer contain the - * the RTP header extension for `abs-capture-time`. - */ - public SenderLatencyMs: number | undefined = undefined; + /* The round trip time (milliseconds) between each sender->receiver->sender */ + public RTTMs: number | undefined = undefined; - /* The round trip time (milliseconds) between each sender->receiver->sender */ - public RTTMs: number | undefined = undefined; + /* Average time taken (milliseconds) from video packet receipt to post-decode. */ + public AverageProcessingDelayMs: number | undefined = undefined; - /* Average time taken (milliseconds) from video packet receipt to post-decode. */ - public AverageProcessingDelayMs: number | undefined = undefined; + /* Average time taken (milliseconds) inside the jitter buffer (which is post-receipt but pre-decode). */ + public AverageJitterBufferDelayMs: number | undefined = undefined; - /* Average time taken (milliseconds) inside the jitter buffer (which is post-receipt but pre-decode). */ - public AverageJitterBufferDelayMs: number | undefined = undefined; + /* Average time taken (milliseconds) to decode a video frame. */ + public AverageDecodeLatencyMs: number | undefined = undefined; - /* Average time taken (milliseconds) to decode a video frame. */ - public AverageDecodeLatencyMs: number | undefined = undefined; + /* Average time taken (milliseconds) to between receipt of the first and last video packet of a. */ + public AverageAssemblyDelayMs: number | undefined = undefined; - /* Average time taken (milliseconds) to between receipt of the first and last video packet of a. */ - public AverageAssemblyDelayMs: number | undefined = undefined; - - /* The sender latency + RTT/2 + processing delay */ - public AverageE2ELatency: number | undefined = undefined; -} \ No newline at end of file + /* The sender latency + RTT/2 + processing delay */ + public AverageE2ELatency: number | undefined = undefined; +} diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 5975aaeef..39445e531 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -100,7 +100,6 @@ export class PeerConnectionController { Logger.Info('Receive Offer'); this.peerConnection?.setRemoteDescription(offer).then(() => { - // Fire event for when remote offer description is set this.onSetRemoteDescription(offer); @@ -176,7 +175,10 @@ export class PeerConnectionController { this.onVideoStats(this.aggregatedStats); // Calculate latency using stats and video receivers and then call the handling function - let latencyInfo: LatencyInfo = this.latencyCalculator.calculate(this.aggregatedStats, this.peerConnection.getReceivers()); + const latencyInfo: LatencyInfo = this.latencyCalculator.calculate( + this.aggregatedStats, + this.peerConnection.getReceivers() + ); this.onLatencyCalculated(latencyInfo); // Update the preferred codec selection based on what was actually negotiated diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index b64386100..561abcdad 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -99,7 +99,7 @@ export class WebRtcSdpAnswerEvent extends Event { /** The sdp answer */ sdp: RTCSessionDescriptionInit; }; - constructor(data: WebRtcSdpAnswerEvent["data"]) { + constructor(data: WebRtcSdpAnswerEvent['data']) { super('webRtcSdpAnswer'); this.data = data; } @@ -114,7 +114,7 @@ export class WebRtcSdpOfferEvent extends Event { /** The sdp offer */ sdp: RTCSessionDescriptionInit; }; - constructor(data: WebRtcSdpOfferEvent["data"]) { + constructor(data: WebRtcSdpOfferEvent['data']) { super('webRtcSdpOffer'); this.data = data; } @@ -419,7 +419,7 @@ export class LatencyCalculatedEvent extends Event { readonly type: 'latencyCalculated'; readonly data: { latencyInfo: LatencyInfo; - } + }; constructor(data: LatencyCalculatedEvent['data']) { super('latencyCalculated'); this.data = data; diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index 54b3db87a..aed9ffb7a 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -1009,22 +1009,22 @@ export class WebRtcPlayerController { // set up peer connection controller video stats this.peerConnectionController.onVideoStats = (event: AggregatedStats) => { this.handleVideoStats(event); - } + }; /* Set event handler for latency information is calculated, handle the event by propogating to the PixelStreaming API */ this.peerConnectionController.onLatencyCalculated = (latencyInfo: LatencyInfo) => { this.pixelStreaming._onLatencyCalculated(latencyInfo); - } + }; /* When the Peer Connection wants to send an offer have it handled */ this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => { this.handleSendWebRTCOffer(offer); - } + }; /* Set event handler for when local answer description is set */ this.peerConnectionController.onSetLocalDescription = (answer: RTCSessionDescriptionInit) => { this.handleSendWebRTCAnswer(answer); - } + }; /* Set event handler for when remote offer description is set */ this.peerConnectionController.onSetRemoteDescription = (offer: RTCSessionDescriptionInit) => { diff --git a/Frontend/library/src/pixelstreamingfrontend.ts b/Frontend/library/src/pixelstreamingfrontend.ts index e93aade3b..f28f57d98 100644 --- a/Frontend/library/src/pixelstreamingfrontend.ts +++ b/Frontend/library/src/pixelstreamingfrontend.ts @@ -43,7 +43,7 @@ export { CandidateStat } from './PeerConnectionController/CandidateStat'; export { DataChannelStats } from './PeerConnectionController/DataChannelStats'; export { InboundAudioStats, InboundVideoStats } from './PeerConnectionController/InboundRTPStats'; export { OutBoundVideoStats } from './PeerConnectionController/OutBoundRTPStats'; -export * from "./PeerConnectionController/LatencyCalculator" +export * from './PeerConnectionController/LatencyCalculator'; export * from './DataChannel/DataChannelLatencyTestResults'; export * from './Util/EventEmitter'; export * from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; diff --git a/Frontend/ui-library/src/UI/LatencyTest.ts b/Frontend/ui-library/src/UI/LatencyTest.ts index 197850706..2c8b111b5 100644 --- a/Frontend/ui-library/src/UI/LatencyTest.ts +++ b/Frontend/ui-library/src/UI/LatencyTest.ts @@ -71,40 +71,46 @@ export class LatencyTest { Logger.Info(JSON.stringify(latencyTimings)); let latencyStatsInnerHTML = ''; - if(latencyTimings.networkLatency !== undefined && latencyTimings.networkLatency > 0) { + if (latencyTimings.networkLatency !== undefined && latencyTimings.networkLatency > 0) { latencyStatsInnerHTML += '
Net latency RTT (ms): ' + latencyTimings.networkLatency + '
'; } - if(latencyTimings.EncodeMs !== undefined && latencyTimings.EncodeMs > 0) { + if (latencyTimings.EncodeMs !== undefined && latencyTimings.EncodeMs > 0) { latencyStatsInnerHTML += '
UE Encode (ms): ' + latencyTimings.EncodeMs + '
'; } - if(latencyTimings.CaptureToSendMs !== undefined && latencyTimings.CaptureToSendMs > 0) { + if (latencyTimings.CaptureToSendMs !== undefined && latencyTimings.CaptureToSendMs > 0) { latencyStatsInnerHTML += '
UE Capture (ms): ' + latencyTimings.CaptureToSendMs + '
'; } - if(latencyTimings.browserSendLatency !== undefined && latencyTimings.browserSendLatency > 0) { - latencyStatsInnerHTML += '
Browser send latency (ms): ' + latencyTimings.browserSendLatency + '
'; + if (latencyTimings.browserSendLatency !== undefined && latencyTimings.browserSendLatency > 0) { + latencyStatsInnerHTML += + '
Browser send latency (ms): ' + latencyTimings.browserSendLatency + '
'; } - if(latencyTimings.frameDisplayDeltaTimeMs !== undefined && latencyTimings.browserReceiptTimeMs !== undefined) { + if ( + latencyTimings.frameDisplayDeltaTimeMs !== undefined && + latencyTimings.browserReceiptTimeMs !== undefined + ) { latencyStatsInnerHTML += - latencyTimings.frameDisplayDeltaTimeMs && latencyTimings.browserReceiptTimeMs - ? '
Browser receive latency (ms): ' + latencyTimings.frameDisplayDeltaTimeMs + '
' - : ''; + latencyTimings.frameDisplayDeltaTimeMs && latencyTimings.browserReceiptTimeMs + ? '
Browser receive latency (ms): ' + + latencyTimings.frameDisplayDeltaTimeMs + + '
' + : ''; } - if(latencyTimings.latencyExcludingDecode !== undefined) { + if (latencyTimings.latencyExcludingDecode !== undefined) { latencyStatsInnerHTML += - '
Total latency (excluding browser) (ms): ' + - latencyTimings.latencyExcludingDecode + - '
'; + '
Total latency (excluding browser) (ms): ' + + latencyTimings.latencyExcludingDecode + + '
'; } - if(latencyTimings.endToEndLatency !== undefined) { + if (latencyTimings.endToEndLatency !== undefined) { latencyStatsInnerHTML += latencyTimings.endToEndLatency - ? '
Total latency (ms): ' + latencyTimings.endToEndLatency + '
' - : ''; + ? '
Total latency (ms): ' + latencyTimings.endToEndLatency + '
' + : ''; } this.latencyTestResultsElement.innerHTML = latencyStatsInnerHTML; diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index 5199f9026..d9b56f0a7 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -311,7 +311,10 @@ export class StatsPanel { ); // QP - if(stats.sessionStats.videoEncoderAvgQP !== undefined && !Number.isNaN(stats.sessionStats.videoEncoderAvgQP)) { + if ( + stats.sessionStats.videoEncoderAvgQP !== undefined && + !Number.isNaN(stats.sessionStats.videoEncoderAvgQP) + ) { this.addOrUpdateStat( 'QPStat', 'Video quantization parameter', @@ -323,7 +326,7 @@ export class StatsPanel { } public handleLatencyInfo(latencyInfo: LatencyInfo) { - if(latencyInfo.SenderLatencyMs !== undefined && latencyInfo.SenderLatencyMs > 0) { + if (latencyInfo.SenderLatencyMs !== undefined && latencyInfo.SenderLatencyMs > 0) { this.addOrUpdateStat( 'SenderSideLatency', 'Sender latency (ms)', @@ -331,7 +334,7 @@ export class StatsPanel { ); } - if(latencyInfo.AverageAssemblyDelayMs !== undefined && latencyInfo.AverageAssemblyDelayMs > 0) { + if (latencyInfo.AverageAssemblyDelayMs !== undefined && latencyInfo.AverageAssemblyDelayMs > 0) { this.addOrUpdateStat( 'AvgAssemblyDelay', 'Assembly delay (ms)', @@ -339,7 +342,7 @@ export class StatsPanel { ); } - if(latencyInfo.AverageDecodeLatencyMs !== undefined && latencyInfo.AverageDecodeLatencyMs > 0) { + if (latencyInfo.AverageDecodeLatencyMs !== undefined && latencyInfo.AverageDecodeLatencyMs > 0) { this.addOrUpdateStat( 'AvgDecodeDelay', 'Decode time (ms)', @@ -347,7 +350,10 @@ export class StatsPanel { ); } - if(latencyInfo.AverageJitterBufferDelayMs !== undefined && latencyInfo.AverageJitterBufferDelayMs > 0) { + if ( + latencyInfo.AverageJitterBufferDelayMs !== undefined && + latencyInfo.AverageJitterBufferDelayMs > 0 + ) { this.addOrUpdateStat( 'AvgJitterBufferDelay', 'Jitter buffer (ms)', @@ -355,7 +361,7 @@ export class StatsPanel { ); } - if(latencyInfo.AverageProcessingDelayMs !== undefined && latencyInfo.AverageProcessingDelayMs > 0) { + if (latencyInfo.AverageProcessingDelayMs !== undefined && latencyInfo.AverageProcessingDelayMs > 0) { this.addOrUpdateStat( 'AvgProcessingDelay', 'Processing delay (ms)', @@ -363,7 +369,7 @@ export class StatsPanel { ); } - if(latencyInfo.AverageE2ELatency !== undefined && latencyInfo.AverageE2ELatency > 0) { + if (latencyInfo.AverageE2ELatency !== undefined && latencyInfo.AverageE2ELatency > 0) { this.addOrUpdateStat( 'AvgE2ELatency', 'Total latency (ms)', From 56ae963b02e797139cc28dde37982637e75eb7cf Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 24 Jan 2025 14:29:39 +1000 Subject: [PATCH 11/27] Fix failing unit test --- .../library/src/PeerConnectionController/AggregatedStats.ts | 5 +++++ .../src/PeerConnectionController/LatencyCalculator.ts | 2 +- Frontend/library/src/__test__/mockRTCPeerConnection.ts | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index a2123684e..7e711f264 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -288,6 +288,11 @@ export class AggregatedStats { * @returns The candidate pair that is currently receiving data */ public getActiveCandidatePair(): CandidatePairStats | null { + + if(this.candidatePairs === undefined) { + return null; + } + // Check if the RTCTransport stat is not undefined if (this.transportStats) { // Return the candidate pair that matches the transport candidate pair id diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts index 67066d77d..dab5e20ef 100644 --- a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -27,7 +27,7 @@ export class LatencyCalculator { const activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); if ( - activeCandidatePair !== null && + !!activeCandidatePair && activeCandidatePair.currentRoundTripTime !== undefined && activeCandidatePair.currentRoundTripTime > 0 ) { diff --git a/Frontend/library/src/__test__/mockRTCPeerConnection.ts b/Frontend/library/src/__test__/mockRTCPeerConnection.ts index a7935e037..1e581c044 100644 --- a/Frontend/library/src/__test__/mockRTCPeerConnection.ts +++ b/Frontend/library/src/__test__/mockRTCPeerConnection.ts @@ -111,7 +111,7 @@ export class MockRTCPeerConnectionImpl implements RTCPeerConnection { throw new Error("Method not implemented."); } getReceivers(): RTCRtpReceiver[] { - throw new Error("Method not implemented."); + return []; } getSenders(): RTCRtpSender[] { throw new Error("Method not implemented."); From 8ef474850afd36a15f5d4f8d7feed612ad21dbd3 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 24 Jan 2025 14:37:59 +1000 Subject: [PATCH 12/27] Fix linting --- .../library/src/PeerConnectionController/AggregatedStats.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 7e711f264..83e93dbfa 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -288,8 +288,7 @@ export class AggregatedStats { * @returns The candidate pair that is currently receiving data */ public getActiveCandidatePair(): CandidatePairStats | null { - - if(this.candidatePairs === undefined) { + if (this.candidatePairs === undefined) { return null; } From 6e85d6694491b1fd48ba808f0a8f0f0a739e7e13 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 24 Jan 2025 15:47:01 +1000 Subject: [PATCH 13/27] Attempt to fix test race condition through explicitly adding listener before starting the stream --- .../FrontendTests/tests/basic_stream.spec.ts | 21 +++++++++------- Extras/FrontendTests/tests/helpers.ts | 11 ++++++++- Extras/FrontendTests/tests/keyboard.spec.ts | 4 ++-- Extras/FrontendTests/tests/mouse.spec.ts | 18 +++++--------- .../tests/peerconnection.spec.ts | 11 +++------ .../tests/resolution_changes.spec.ts | 8 +++---- .../tests/stream_test.spec.ts | 24 ++++++++++++------- 7 files changed, 52 insertions(+), 45 deletions(-) diff --git a/Extras/FrontendTests/tests/basic_stream.spec.ts b/Extras/FrontendTests/tests/basic_stream.spec.ts index d1fd212da..c715a7b3a 100644 --- a/Extras/FrontendTests/tests/basic_stream.spec.ts +++ b/Extras/FrontendTests/tests/basic_stream.spec.ts @@ -1,6 +1,7 @@ import { test } from './fixtures'; import { expect } from './matchers'; import * as helpers from './helpers'; +import { StatsReceivedEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; // NOTE: add a new test to check qp values @@ -10,20 +11,22 @@ test('Test default stream.', { }, async ({ page, streamerId }) => { await page.goto(`/?StreamerId=${streamerId}`); - await page.getByText('Click to start').click(); // let the stream run for a short duration - await helpers.waitForVideo(page); - await helpers.delay(1000); - - // query the frontend for its calculated stats - const frame_count:number = await page.evaluate(()=> { - let videoStats = pixelStreaming._webRtcController.peerConnectionController.aggregatedStats.inboundVideoStats; - return videoStats.framesReceived; + await helpers.startAndWaitForVideo(page); + + let frameCount: number = await page.evaluate(()=> { + return new Promise((resolve) => { + window.pixelStreaming.addEventListener("statsReceived", (e: StatsReceivedEvent) => { + if(e.data.aggregatedStats && e.data.aggregatedStats.inboundVideoStats && e.data.aggregatedStats.inboundVideoStats.framesReceived) { + resolve(e.data.aggregatedStats.inboundVideoStats.framesReceived); + } + }); + }); }); // pass the test if we recorded any frames - expect(frame_count).toBeGreaterThan(0); + expect(frameCount).toBeGreaterThan(0); }); diff --git a/Extras/FrontendTests/tests/helpers.ts b/Extras/FrontendTests/tests/helpers.ts index bc39a490f..008cb046b 100644 --- a/Extras/FrontendTests/tests/helpers.ts +++ b/Extras/FrontendTests/tests/helpers.ts @@ -31,12 +31,21 @@ export function delay(time: number) { }); } -export async function waitForVideo(page: Page) { +export async function startStreaming(page: Page) { + await page.evaluate(()=> { + window.pixelStreaming.connect(); + }); +} + +export async function startAndWaitForVideo(page: Page) { await page.evaluate(()=> { return new Promise((resolve) => { + // Note: Assign listener before we start the connection window.pixelStreaming.addEventListener('playStream', (event) => { return resolve(event); }); + // Make the actual connection initiation + window.pixelStreaming.connect(); }); }); } diff --git a/Extras/FrontendTests/tests/keyboard.spec.ts b/Extras/FrontendTests/tests/keyboard.spec.ts index 3b40c9f66..c483e016e 100644 --- a/Extras/FrontendTests/tests/keyboard.spec.ts +++ b/Extras/FrontendTests/tests/keyboard.spec.ts @@ -12,8 +12,8 @@ test('Test keyboard events', { }, async ({ page, streamerPage, streamerId }) => { await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + + await helpers.startAndWaitForVideo(page); const playerBox = await page.locator('#videoElementParent').boundingBox(); expect(playerBox).not.toBeNull(); diff --git a/Extras/FrontendTests/tests/mouse.spec.ts b/Extras/FrontendTests/tests/mouse.spec.ts index ba56846e1..3d80a98bc 100644 --- a/Extras/FrontendTests/tests/mouse.spec.ts +++ b/Extras/FrontendTests/tests/mouse.spec.ts @@ -22,9 +22,8 @@ test('Test mouse enter/leave', { // }); await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true&HoveringMouse=true`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); // reduce the size of the window so we can leave await page.setViewportSize({ width: 100, height: 100 }); @@ -66,9 +65,8 @@ test('Test mouse wheel', { // }); await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true&HoveringMouse=false`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); const playerBox = await page.locator('#videoElementParent').boundingBox(); expect(playerBox).not.toBeNull(); @@ -115,9 +113,8 @@ test('Test locked mouse movement', { }); await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true&HoveringMouse=false`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); const playerBox = await page.locator('#videoElementParent').boundingBox(); expect(playerBox).not.toBeNull(); @@ -193,9 +190,8 @@ test('Test hovering mouse movement', { // }); await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true&HoveringMouse=true`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); const playerBox = await page.locator('#videoElementParent').boundingBox(); expect(playerBox).not.toBeNull(); @@ -245,9 +241,8 @@ test('Test mouse input after resizing. Hover mouse.', { // }); await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true&HoveringMouse=true`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); // resize the window to be smaller const oldSize = page.viewportSize(); @@ -309,9 +304,8 @@ test('Test mouse input after resizing. locked mouse.', { // }); await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true&HoveringMouse=false`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); // resize the window to be smaller const oldSize = page.viewportSize(); diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index a01d8d8d1..ba0c9da57 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -36,9 +36,8 @@ test('Test abs-capture-time header extension found for streamer', { }); - await page.getByText('Click to start').click(); + await helpers.startAndWaitForVideo(page); - await helpers.waitForVideo(page); let localDescSdp: RTCSessionDescriptionInit = await localDescription; let remoteDescSdp: RTCSessionDescriptionInit = await getSdpOffer; @@ -77,12 +76,9 @@ test('Test abs-capture-time header extension found in PSInfra frontend', { }); - await page.getByText('Click to start').click(); - + await helpers.startAndWaitForVideo(page); const answer: RTCSessionDescriptionInit = await getSdpAnswer; - await helpers.waitForVideo(page); - expect(answer).toBeDefined(); expect(answer.sdp).toBeDefined(); @@ -108,8 +104,7 @@ test('Test latency calculation', { window.pixelStreaming.config.setFlagEnabled("EnableCaptureTimeExt", true); }); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + await helpers.startAndWaitForVideo(page); // Wait for the latency info event to be fired let latencyInfo: LatencyInfo = await page.evaluate(() => { diff --git a/Extras/FrontendTests/tests/resolution_changes.spec.ts b/Extras/FrontendTests/tests/resolution_changes.spec.ts index 38c71ebb0..8ac72596b 100644 --- a/Extras/FrontendTests/tests/resolution_changes.spec.ts +++ b/Extras/FrontendTests/tests/resolution_changes.spec.ts @@ -13,8 +13,8 @@ test('Test resolution changes with match viewport on.', { // first with match viewport enabled await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=true`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + + await helpers.startAndWaitForVideo(page); const events = await getEventsFor(streamerPage, async () => { await page.setViewportSize({ width: 100, height: 100 }); @@ -39,8 +39,8 @@ test('Test resolution changes with match viewport off.', { // first with match viewport enabled await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=false`); - await page.getByText('Click to start').click(); - await helpers.waitForVideo(page); + + await helpers.startAndWaitForVideo(page); await page.click("#streamingVideo"); const events = await getEventsFor(streamerPage, async () => { diff --git a/Extras/MinimalStreamTester/tests/stream_test.spec.ts b/Extras/MinimalStreamTester/tests/stream_test.spec.ts index 1e8cbe7fa..09db3e352 100644 --- a/Extras/MinimalStreamTester/tests/stream_test.spec.ts +++ b/Extras/MinimalStreamTester/tests/stream_test.spec.ts @@ -7,12 +7,14 @@ function delay(time: number) { }); } -async function waitForVideo(page: Page) { +async function startAndWaitForVideo(page: Page) { await page.evaluate(()=> { return new Promise((resolve) => { - pixelStreaming.addEventListener('playStream', (event) => { + window.pixelStreaming.addEventListener('playStream', (event) => { return resolve(event); }); + // Start the stream now we have listener attached + window.pixelStreaming.connect(); }); }); } @@ -22,20 +24,24 @@ test('Test default stream.', async ({ page }, testinfo) => { // set a long timeout to allow for slow software rendering test.setTimeout(2 * 60 * 1000); - + await page.goto("/?StreamerId=DefaultStreamer"); - await page.getByText('Click to start').click(); // wait until we get a stream - await waitForVideo(page); + await startAndWaitForVideo(page); // let the stream run for a small duration await delay(15000); // query the frontend for its calculated stats - const frame_count:number = await page.evaluate(()=> { - let videoStats = pixelStreaming._webRtcController.peerConnectionController.aggregatedStats.inboundVideoStats; - return videoStats.framesReceived; + let frameCount: number = await page.evaluate(()=> { + return new Promise((resolve) => { + window.pixelStreaming.addEventListener("statsReceived", (e) => { + if(e.data.aggregatedStats && e.data.aggregatedStats.inboundVideoStats && e.data.aggregatedStats.inboundVideoStats.framesReceived) { + resolve(e.data.aggregatedStats.inboundVideoStats.framesReceived); + } + }); + }); }); // take a screenshot for posterity @@ -47,6 +53,6 @@ test('Test default stream.', async ({ page }, testinfo) => { testinfo.attach('screenshot', { body: screenshot, contentType: 'image/png' }); // pass the test if we recorded any frames - expect(frame_count).toBeGreaterThan(0); + expect(frameCount).toBeGreaterThan(0); }); From 47f041c5091174e68e9bf8b9e6bf60ebfbdeda71 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 24 Jan 2025 16:08:45 +1000 Subject: [PATCH 14/27] Attempt to update playwright version used in minimal stream tester --- Extras/MinimalStreamTester/package.json | 2 +- package-lock.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Extras/MinimalStreamTester/package.json b/Extras/MinimalStreamTester/package.json index f681a5782..8d961b9e7 100644 --- a/Extras/MinimalStreamTester/package.json +++ b/Extras/MinimalStreamTester/package.json @@ -10,7 +10,7 @@ "author": "", "license": "ISC", "devDependencies": { - "@playwright/test": "^1.49.0", + "@playwright/test": "^1.49.1", "@types/node": "^20.12.7", "@types/uuid": "^9.0.8" }, diff --git a/package-lock.json b/package-lock.json index df536e2e3..664e925a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3626,7 +3626,7 @@ "uuid": "^9.0.0" }, "devDependencies": { - "@playwright/test": "^1.49.0", + "@playwright/test": "^1.49.1", "@types/node": "^20.12.7", "@types/uuid": "^9.0.8" } From 514089ccd751283423bbd1ccdde1d028473448a2 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 10:38:32 +1000 Subject: [PATCH 15/27] Remove problematic test teardown when test was skipped --- Extras/FrontendTests/tests/fixtures.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index 7a0b73c5f..7a5ef55b1 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -11,11 +11,6 @@ export const test = base.extend({ const streamerPage = await context.newPage(); await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}` + `${process.env.STREAMER_SIGNALLING_URL !== undefined ? '?SignallingURL=' + process.env.STREAMER_SIGNALLING_URL : ""}`); await use(streamerPage); - - // this is called after test is run by `use` - streamerPage.evaluate(() => { - window.streamer.stopStreaming(); - }); }, streamerId: async ({ streamerPage }, use) => { From 2c47b9478f5bbe2aa8d0833a3fbc0e99eea264e1 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 13:19:40 +1000 Subject: [PATCH 16/27] Add extra info step to streaming CI job to help diagnose CI failures --- .github/workflows/healthcheck-streaming.yml | 9 ++++++++- Extras/MinimalStreamTester/docker-compose.yml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/healthcheck-streaming.yml b/.github/workflows/healthcheck-streaming.yml index 679e22bb2..072ef3d33 100644 --- a/.github/workflows/healthcheck-streaming.yml +++ b/.github/workflows/healthcheck-streaming.yml @@ -109,7 +109,7 @@ jobs: - name: Run Streamer working-directory: Streamer - run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Win64-Shipping-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwaRerendering","-PixelStreamingEncoderCodec=vp8" + run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Win64-Shipping-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8" - name: Prepare test working-directory: Extras\MinimalStreamTester @@ -121,9 +121,16 @@ jobs: - name: Wait for signalling to come up run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/api/status + - name: Wait for front end to come up + run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/ + - name: Wait for streamer to come up run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/api/streamers/DefaultStreamer + - name: Output streamer logs + working-directory: Streamer + run: ls ".\Minimal\Saved\Logs\" && cat ".\Minimal\Saved\Logs\Minimal.log" + - name: Test if we can stream working-directory: Extras\MinimalStreamTester run: | diff --git a/Extras/MinimalStreamTester/docker-compose.yml b/Extras/MinimalStreamTester/docker-compose.yml index 426674e4c..71586c387 100644 --- a/Extras/MinimalStreamTester/docker-compose.yml +++ b/Extras/MinimalStreamTester/docker-compose.yml @@ -19,7 +19,7 @@ services: streamer: image: pixelstreamingunofficial/ps-minimal-streamer-linux - command: -PixelStreamingURL=ws://signalling:8888 -nothreadtimeout + command: -PixelStreamingURL=ws://signalling:8888 -nothreadtimeout -PixelStreamingEncoderCodec=vp8 -RenderOffScreen -AllowSoftwareRendering networks: - testing healthcheck: From ee4edc22f5725fc064b8ce552fc379c8c2cf612b Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 14:07:15 +1000 Subject: [PATCH 17/27] Iterating on what to log --- .github/workflows/healthcheck-streaming.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/healthcheck-streaming.yml b/.github/workflows/healthcheck-streaming.yml index 072ef3d33..b478aaad5 100644 --- a/.github/workflows/healthcheck-streaming.yml +++ b/.github/workflows/healthcheck-streaming.yml @@ -129,7 +129,7 @@ jobs: - name: Output streamer logs working-directory: Streamer - run: ls ".\Minimal\Saved\Logs\" && cat ".\Minimal\Saved\Logs\Minimal.log" + run: ls ".\Minimal\" - name: Test if we can stream working-directory: Extras\MinimalStreamTester From 4f1ea36d14fda08bf71d289c1f75ea0a4f4be196 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 14:28:22 +1000 Subject: [PATCH 18/27] Attempt to revert some load bearing changes and add logging to stream test streamer --- .github/workflows/healthcheck-streaming.yml | 4 ++-- Extras/FrontendTests/tests/fixtures.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/healthcheck-streaming.yml b/.github/workflows/healthcheck-streaming.yml index b478aaad5..146dcebee 100644 --- a/.github/workflows/healthcheck-streaming.yml +++ b/.github/workflows/healthcheck-streaming.yml @@ -109,7 +109,7 @@ jobs: - name: Run Streamer working-directory: Streamer - run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Win64-Shipping-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8" + run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Win64-Shipping-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8", "-Log=Minimal.log" - name: Prepare test working-directory: Extras\MinimalStreamTester @@ -129,7 +129,7 @@ jobs: - name: Output streamer logs working-directory: Streamer - run: ls ".\Minimal\" + run: ls ".\Minimal\" && Test-Path ".\Minimal\Saved\Logs\Minimal.log" && cat ".\Minimal\Saved\Logs\Minimal.log" - name: Test if we can stream working-directory: Extras\MinimalStreamTester diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index 7a5ef55b1..0d4d0b54e 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -9,7 +9,7 @@ type PSTestFixtures = { export const test = base.extend({ streamerPage: async ({ context }, use) => { const streamerPage = await context.newPage(); - await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}` + `${process.env.STREAMER_SIGNALLING_URL !== undefined ? '?SignallingURL=' + process.env.STREAMER_SIGNALLING_URL : ""}`); + await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}?SignallingURL=${process.env.STREAMER_SIGNALLING_URL}`); await use(streamerPage); }, streamerId: async ({ streamerPage }, use) => { From df82e74b932006bf41380b465fc1b26835e9f654 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 15:12:02 +1000 Subject: [PATCH 19/27] Move to using latest minimal streamer release --- .github/workflows/healthcheck-streaming.yml | 8 +++---- .../tests/resolution_changes.spec.ts | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/healthcheck-streaming.yml b/.github/workflows/healthcheck-streaming.yml index 146dcebee..c2b71781d 100644 --- a/.github/workflows/healthcheck-streaming.yml +++ b/.github/workflows/healthcheck-streaming.yml @@ -60,11 +60,11 @@ jobs: uses: robinraju/release-downloader@v1 with: repository: 'EpicGamesExt/PixelStreamingInfrastructure' - tag: 'minimal-streamer' - fileName: 'Minimal-PixelStreamer-5.5.7z' + tag: 'minimal-streamer-5.5' + fileName: 'Minimal-PixelStreamer-5.5-Win64-Development.7z' - name: Extract streamer - run: 7z x -oStreamer Minimal-PixelStreamer-5.5.7z + run: 7z x -oStreamer Minimal-PixelStreamer-5.5-Win64-Development.7z - name: Build Common working-directory: Common @@ -109,7 +109,7 @@ jobs: - name: Run Streamer working-directory: Streamer - run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Win64-Shipping-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8", "-Log=Minimal.log" + run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8", "-Log=Minimal.log" - name: Prepare test working-directory: Extras\MinimalStreamTester diff --git a/Extras/FrontendTests/tests/resolution_changes.spec.ts b/Extras/FrontendTests/tests/resolution_changes.spec.ts index 8ac72596b..53bddedb9 100644 --- a/Extras/FrontendTests/tests/resolution_changes.spec.ts +++ b/Extras/FrontendTests/tests/resolution_changes.spec.ts @@ -41,22 +41,27 @@ test('Test resolution changes with match viewport off.', { await page.goto(`/?StreamerId=${streamerId}&MatchViewportRes=false`); await helpers.startAndWaitForVideo(page); - await page.click("#streamingVideo"); const events = await getEventsFor(streamerPage, async () => { + // We do a click here so we have some player input to send the streamer + await page.click("#streamingVideo"); await page.setViewportSize({ width: 100, height: 100 }); await helpers.delay(1000); await page.setViewportSize({ width: 800, height: 600 }); await helpers.delay(1000); }); - const firstPlayerId = Object.keys(events)[0]; - const singlePlayerEvents = events[firstPlayerId]; - const expectedActions: DataChannelEvent[] = [ - { type: PSEventTypes.Command, command: '{\"Resolution.Width\":100,\"Resolution.Height\":100}' }, - { type: PSEventTypes.Command, command: '{\"Resolution.Width\":800,\"Resolution.Height\":600}' }, - ]; - expect(singlePlayerEvents).not.toContainActions(expectedActions); + const firstPlayerId = Object.keys(events).length > 0 ? Object.keys(events)[0] : null; + if(firstPlayerId != null) { + const singlePlayerEvents = events[firstPlayerId]; + if(singlePlayerEvents.length > 0) { + const expectedActions: DataChannelEvent[] = [ + { type: PSEventTypes.Command, command: '{\"Resolution.Width\":100,\"Resolution.Height\":100}' }, + { type: PSEventTypes.Command, command: '{\"Resolution.Width\":800,\"Resolution.Height\":600}' }, + ]; + expect(singlePlayerEvents).not.toContainActions(expectedActions); + } + } }); From c5c46e004a26937b87e88a1f94260c1aa26dd77d Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 17:57:19 +1000 Subject: [PATCH 20/27] Disable header extension for Firefox as it causes it to not connect --- .github/workflows/healthcheck-streaming.yml | 2 +- Common/src/Util/SdpUtils.ts | 5 +++-- Extras/FrontendTests/playwright.config.ts | 4 ++-- Extras/FrontendTests/tests/fixtures.ts | 2 +- .../tests/peerconnection.spec.ts | 14 ++++++++++-- Extras/JSStreamer/src/streamer.ts | 13 +++++++---- .../PeerConnectionController.ts | 22 ++++++++++++++++--- 7 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.github/workflows/healthcheck-streaming.yml b/.github/workflows/healthcheck-streaming.yml index c2b71781d..05f89b595 100644 --- a/.github/workflows/healthcheck-streaming.yml +++ b/.github/workflows/healthcheck-streaming.yml @@ -109,7 +109,7 @@ jobs: - name: Run Streamer working-directory: Streamer - run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8", "-Log=Minimal.log" + run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-resx=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8", "-Log=Minimal.log" - name: Prepare test working-directory: Extras\MinimalStreamTester diff --git a/Common/src/Util/SdpUtils.ts b/Common/src/Util/SdpUtils.ts index 9393cabf7..dc5090a77 100644 --- a/Common/src/Util/SdpUtils.ts +++ b/Common/src/Util/SdpUtils.ts @@ -1,7 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. export class SDPUtils { - static addHeaderExtensionToSdp(sdp: string, uri: string): string { + static addVideoHeaderExtensionToSdp(sdp: string, uri: string): string { // Find the highest used header extension id by sorting the extension ids used, // eliminating duplicates and adding one. // Todo: Update this when WebRTC in Chrome supports the header extension API. @@ -18,6 +18,7 @@ export class SDPUtils { return (index > 0 ? 'm=' + part : part).trim() + '\r\n'; }); const sessionPart = sections.shift(); - return sessionPart + sections.map((mediaSection) => mediaSection + extmapLine).join(''); + // Only add extension to m=video media section + return sessionPart + sections.map((mediaSection) => mediaSection.startsWith("m=video") ? mediaSection + extmapLine : mediaSection).join(''); } } diff --git a/Extras/FrontendTests/playwright.config.ts b/Extras/FrontendTests/playwright.config.ts index 429f8e95c..2c0248afb 100755 --- a/Extras/FrontendTests/playwright.config.ts +++ b/Extras/FrontendTests/playwright.config.ts @@ -14,8 +14,8 @@ export default defineConfig({ forbidOnly: !!process.env.CI, /* Retry on CI only */ retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : 5, + /* Opt out of parallel tests in general as multiple streamers mean they can connect to the wrong test */ + workers: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [['html', { open: 'never' }]], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ diff --git a/Extras/FrontendTests/tests/fixtures.ts b/Extras/FrontendTests/tests/fixtures.ts index 0d4d0b54e..7a5ef55b1 100644 --- a/Extras/FrontendTests/tests/fixtures.ts +++ b/Extras/FrontendTests/tests/fixtures.ts @@ -9,7 +9,7 @@ type PSTestFixtures = { export const test = base.extend({ streamerPage: async ({ context }, use) => { const streamerPage = await context.newPage(); - await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}?SignallingURL=${process.env.STREAMER_SIGNALLING_URL}`); + await streamerPage.goto(`${process.env.PIXELSTREAMER_URL || 'http://localhost:4000'}` + `${process.env.STREAMER_SIGNALLING_URL !== undefined ? '?SignallingURL=' + process.env.STREAMER_SIGNALLING_URL : ""}`); await use(streamerPage); }, streamerId: async ({ streamerPage }, use) => { diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index ba0c9da57..81ef0da57 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -5,7 +5,12 @@ import { Flags, PixelStreaming, WebRtcSdpAnswerEvent, WebRtcSdpOfferEvent, Laten test('Test abs-capture-time header extension found for streamer', { tag: ['@capture-time'], -}, async ({ page, streamerPage, streamerId }) => { +}, async ({ page, streamerPage, streamerId, browserName }) => { + + if(browserName !== 'chromium') { + // Chrome based browsers are the only ones that support. + test.skip(); + } const localDescription: Promise = new Promise(async (resolve) => { @@ -51,7 +56,12 @@ test('Test abs-capture-time header extension found for streamer', { test('Test abs-capture-time header extension found in PSInfra frontend', { tag: ['@capture-time'], -}, async ({ page, streamerPage, streamerId }) => { +}, async ({ page, streamerPage, streamerId, browserName }) => { + + if(browserName !== 'chromium') { + // Chrome based browsers are the only ones that support. + test.skip(); + } await page.goto(`/?StreamerId=${streamerId}`); diff --git a/Extras/JSStreamer/src/streamer.ts b/Extras/JSStreamer/src/streamer.ts index c598b021d..cd05ff955 100644 --- a/Extras/JSStreamer/src/streamer.ts +++ b/Extras/JSStreamer/src/streamer.ts @@ -32,6 +32,7 @@ interface WebRTCSettings { MaxBitrate: number; LowQP: number; HighQP: number; + AbsCaptureTimeHeaderExt: boolean } interface Settings { @@ -90,7 +91,8 @@ export class Streamer extends EventEmitter { MinBitrate: 100000, MaxBitrate: 100000000, LowQP: 25, - HighQP: 37 + HighQP: 37, + AbsCaptureTimeHeaderExt: true }, ConfigOptions: {} }; @@ -309,9 +311,12 @@ export class Streamer extends EventEmitter { } mungeOffer(offerSDP: string) : string { - // Add the abs-capture-time header extension to the sdp extmap - const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; - return SDPUtils.addHeaderExtensionToSdp(offerSDP, kAbsCaptureTime); + if(this.settings.WebRTC.AbsCaptureTimeHeaderExt) { + // Add the abs-capture-time header extension to the sdp extmap + const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; + return SDPUtils.addVideoHeaderExtensionToSdp(offerSDP, kAbsCaptureTime); + } + return offerSDP; } sendDataProtocol(playerId: string) { diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 39445e531..4c2f021f8 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -99,6 +99,14 @@ export class PeerConnectionController { async receiveOffer(offer: RTCSessionDescriptionInit, config: Config) { Logger.Info('Receive Offer'); + // If UE or JSStreamer did send abs-capture-time RTP header extension to a non-Chrome browser + // then remove it from the SDP because if Firefox detects it in offer or answer it will fail to connect + // due having 15 or more header extensions: https://mailarchive.ietf.org/arch/msg/rtcweb/QRnWNuWzGuLRovWdHkodNP6VOgg/ + if(!this.isChromeBased()) { + // example: a=extmap:15 http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time + offer.sdp = offer.sdp.replace(/^a=extmap:\d+ http:\/\/www\.webrtc\.org\/experiments\/rtp-hdrext\/abs-capture-time\r\n/gm,''); + } + this.peerConnection?.setRemoteDescription(offer).then(() => { // Fire event for when remote offer description is set this.onSetRemoteDescription(offer); @@ -254,14 +262,22 @@ export class PeerConnectionController { // We use the line 'useinbandfec=1' (which Opus uses) to set our Opus specific audio parameters. mungedSDP = mungedSDP.replace('useinbandfec=1', audioSDP); - // Add abs-capture-time RTP header extension if we have enabled the setting - if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt)) { - mungedSDP = SDPUtils.addHeaderExtensionToSdp(mungedSDP, kAbsCaptureTime); + // Add abs-capture-time RTP header extension if we have enabled the setting. + // Note: As at Feb 2025, Chromium based browsers are the only ones that support this and + // munging it into the answer in Firefox will cause the connection to fail. + if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt) && this.isChromeBased()) { + mungedSDP = SDPUtils.addVideoHeaderExtensionToSdp(mungedSDP, kAbsCaptureTime); } return mungedSDP; } + isChromeBased() : boolean { + const browserWindow: any = window; + const isChromeLike = !!browserWindow.chrome && (navigator.userAgent.includes("Chromium") || !!browserWindow.chrome.webstore || !!browserWindow.chrome.runtime); + return isChromeLike; + } + /** * When a Ice Candidate is received add to the RTC Peer Connection * @param iceCandidate - RTC Ice Candidate from the Signaling Server From 4b4db38d9003399effc7cdd14440cf606659e59d Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Wed, 29 Jan 2025 18:06:06 +1000 Subject: [PATCH 21/27] Made code more exclusive of Firefox weirdness --- .../PeerConnectionController.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 4c2f021f8..7ed629613 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -102,7 +102,7 @@ export class PeerConnectionController { // If UE or JSStreamer did send abs-capture-time RTP header extension to a non-Chrome browser // then remove it from the SDP because if Firefox detects it in offer or answer it will fail to connect // due having 15 or more header extensions: https://mailarchive.ietf.org/arch/msg/rtcweb/QRnWNuWzGuLRovWdHkodNP6VOgg/ - if(!this.isChromeBased()) { + if(this.isFirefox()) { // example: a=extmap:15 http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time offer.sdp = offer.sdp.replace(/^a=extmap:\d+ http:\/\/www\.webrtc\.org\/experiments\/rtp-hdrext\/abs-capture-time\r\n/gm,''); } @@ -265,17 +265,15 @@ export class PeerConnectionController { // Add abs-capture-time RTP header extension if we have enabled the setting. // Note: As at Feb 2025, Chromium based browsers are the only ones that support this and // munging it into the answer in Firefox will cause the connection to fail. - if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt) && this.isChromeBased()) { + if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt) && !this.isFirefox()) { mungedSDP = SDPUtils.addVideoHeaderExtensionToSdp(mungedSDP, kAbsCaptureTime); } return mungedSDP; } - isChromeBased() : boolean { - const browserWindow: any = window; - const isChromeLike = !!browserWindow.chrome && (navigator.userAgent.includes("Chromium") || !!browserWindow.chrome.webstore || !!browserWindow.chrome.runtime); - return isChromeLike; + isFirefox() : boolean { + return typeof (window as any).InstallTrigger !== "undefined"; } /** From 1833e62c688e69d5f428ad284e413bd0246e0592 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Thu, 30 Jan 2025 08:18:16 +1000 Subject: [PATCH 22/27] Linting fix --- Common/src/Util/SdpUtils.ts | 9 ++++++++- .../PeerConnectionController.ts | 11 +++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Common/src/Util/SdpUtils.ts b/Common/src/Util/SdpUtils.ts index dc5090a77..1a2a4ffe2 100644 --- a/Common/src/Util/SdpUtils.ts +++ b/Common/src/Util/SdpUtils.ts @@ -19,6 +19,13 @@ export class SDPUtils { }); const sessionPart = sections.shift(); // Only add extension to m=video media section - return sessionPart + sections.map((mediaSection) => mediaSection.startsWith("m=video") ? mediaSection + extmapLine : mediaSection).join(''); + return ( + sessionPart + + sections + .map((mediaSection) => + mediaSection.startsWith('m=video') ? mediaSection + extmapLine : mediaSection + ) + .join('') + ); } } diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 7ed629613..91700c431 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -102,9 +102,12 @@ export class PeerConnectionController { // If UE or JSStreamer did send abs-capture-time RTP header extension to a non-Chrome browser // then remove it from the SDP because if Firefox detects it in offer or answer it will fail to connect // due having 15 or more header extensions: https://mailarchive.ietf.org/arch/msg/rtcweb/QRnWNuWzGuLRovWdHkodNP6VOgg/ - if(this.isFirefox()) { + if (this.isFirefox()) { // example: a=extmap:15 http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time - offer.sdp = offer.sdp.replace(/^a=extmap:\d+ http:\/\/www\.webrtc\.org\/experiments\/rtp-hdrext\/abs-capture-time\r\n/gm,''); + offer.sdp = offer.sdp.replace( + /^a=extmap:\d+ http:\/\/www\.webrtc\.org\/experiments\/rtp-hdrext\/abs-capture-time\r\n/gm, + '' + ); } this.peerConnection?.setRemoteDescription(offer).then(() => { @@ -272,8 +275,8 @@ export class PeerConnectionController { return mungedSDP; } - isFirefox() : boolean { - return typeof (window as any).InstallTrigger !== "undefined"; + isFirefox(): boolean { + return typeof (window as any).InstallTrigger !== 'undefined'; } /** From 989a62a99e11c40bcec8b69f8cf47dbb291e3473 Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 31 Jan 2025 16:37:46 +1000 Subject: [PATCH 23/27] Fixing linting error --- .../tests/peerconnection.spec.ts | 26 +-- .../AggregatedStats.ts | 24 ++- .../LatencyCalculator.ts | 200 ++++++++++++++---- .../PeerConnectionController.ts | 22 +- Frontend/ui-library/src/UI/StatsPanel.ts | 88 +++++--- 5 files changed, 264 insertions(+), 96 deletions(-) diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 81ef0da57..0796b713a 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -120,7 +120,7 @@ test('Test latency calculation', { let latencyInfo: LatencyInfo = await page.evaluate(() => { return new Promise((resolve) => { window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => { - if(e.data.latencyInfo && e.data.latencyInfo.SenderLatencyMs) { + if(e.data.latencyInfo && e.data.latencyInfo.senderLatencyMs) { resolve(e.data.latencyInfo); } }); @@ -128,27 +128,27 @@ test('Test latency calculation', { }); expect(latencyInfo).toBeDefined(); - expect(latencyInfo.SenderLatencyMs).toBeDefined(); - expect(latencyInfo.AverageJitterBufferDelayMs).toBeDefined(); - expect(latencyInfo.AverageProcessingDelayMs).toBeDefined(); - expect(latencyInfo.RTTMs).toBeDefined(); - expect(latencyInfo.AverageAssemblyDelayMs).toBeDefined(); - expect(latencyInfo.AverageDecodeLatencyMs).toBeDefined(); + expect(latencyInfo.senderLatencyMs).toBeDefined(); + expect(latencyInfo.averageJitterBufferDelayMs).toBeDefined(); + expect(latencyInfo.averageProcessingDelayMs).toBeDefined(); + expect(latencyInfo.rttMs).toBeDefined(); + expect(latencyInfo.averageAssemblyDelayMs).toBeDefined(); + expect(latencyInfo.averageDecodeLatencyMs).toBeDefined(); // Sender side latency should be less than 500ms in pure CPU test - expect(latencyInfo.SenderLatencyMs).toBeLessThanOrEqual(500) + expect(latencyInfo.senderLatencyMs).toBeLessThanOrEqual(500) // Expect jitter buffer/processing delay to be no greater than 500ms on local link - expect(latencyInfo.AverageJitterBufferDelayMs).toBeLessThanOrEqual(500); - expect(latencyInfo.AverageProcessingDelayMs).toBeLessThanOrEqual(500); + expect(latencyInfo.averageJitterBufferDelayMs).toBeLessThanOrEqual(500); + expect(latencyInfo.averageProcessingDelayMs).toBeLessThanOrEqual(500); // Expect RTT to be less than 10ms on loopback - expect(latencyInfo.RTTMs).toBeLessThanOrEqual(10); + expect(latencyInfo.rttMs).toBeLessThanOrEqual(10); // Expect time to assemble frame from packets to be less than the frame rate itself at 30 fps - expect(latencyInfo.AverageAssemblyDelayMs).toBeLessThanOrEqual(33); + expect(latencyInfo.averageAssemblyDelayMs).toBeLessThanOrEqual(33); // Expect CPU decoder to at least be able to do 30 fps decode - expect(latencyInfo.AverageDecodeLatencyMs).toBeLessThanOrEqual(33); + expect(latencyInfo.averageDecodeLatencyMs).toBeLessThanOrEqual(33); }); diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 83e93dbfa..28af72cc9 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -15,7 +15,6 @@ import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; * The Aggregated Stats that is generated from the RTC Stats Report */ -type RTCStatsTypePS = RTCStatsType | 'stream' | 'media-playout' | 'track'; export class AggregatedStats { inboundVideoStats: InboundVideoStats; inboundAudioStats: InboundAudioStats; @@ -49,7 +48,7 @@ export class AggregatedStats { this.candidatePairs = new Array(); rtcStatsReport.forEach((stat) => { - const type: RTCStatsTypePS = stat.type; + const type: string = stat.type; switch (type) { case 'candidate-pair': @@ -295,13 +294,24 @@ export class AggregatedStats { // Check if the RTCTransport stat is not undefined if (this.transportStats) { // Return the candidate pair that matches the transport candidate pair id - return this.candidatePairs.find( - (candidatePair) => candidatePair.id === this.transportStats.selectedCandidatePairId, - null + const selectedPair: CandidatePairStats | undefined = this.candidatePairs.find( + (candidatePair) => candidatePair.id === this.transportStats.selectedCandidatePairId ); + if (selectedPair === undefined) { + return null; + } else { + return selectedPair; + } } - // Fall back to the selected candidate pair - return this.candidatePairs.find((candidatePair) => candidatePair.selected, null); + // Fall back to the `.selected` member of the candidate pair + const selectedPair: CandidatePairStats | undefined = this.candidatePairs.find( + (candidatePair) => candidatePair.selected + ); + if (selectedPair === undefined) { + return null; + } else { + return selectedPair; + } } } diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts index dab5e20ef..2d10694f1 100644 --- a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -1,6 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. -import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5'; import { AggregatedStats } from './AggregatedStats'; import { CandidatePairStats } from './CandidatePairStats'; @@ -17,14 +16,71 @@ class RTCRtpCaptureSource { senderCaptureTimeOffset: number; } +/** + * FrameTimingInfo is a Chromium-specific set of WebRTC stats useful for latency calculation. It is stored in WebRTC stats as `googTimingFrameInfo`. + * It is defined as an RTP header extension here: https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/video-timing/README.md + * It is defined in source code here: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/video/video_timing.cc;l=82;drc=8d399817282e3c12ed54eb23ec42a5e418298ec6 + * It is discussed by its author here: https://github.com/w3c/webrtc-provisional-stats/issues/40#issuecomment-1272916692 + * In summary it a comma-delimited string that contains the following (in this order): + * 1) RTP timestamp: the RTP timestamp of the frame + * 2) Capture time: timestamp when this frame was captured + * 3) Encode start: timestamp when this frame started to be encoded + * 4) Encode finish: timestamp when this frame finished encoding + * 5) Packetization finish: timestamp when this frame was split into packets and was ready to be sent over the network + * 6) Pacer exit: timestamp when last packet of this frame was sent over the network by the sender at this timestamp + * 7) Network timestamp1: place for the SFU to mark when the frame started being forwarded. Application specific. + * 8) Network timestamp2: place for the SFU to mark when the frame finished being forwarded. Application specific. + * 9) Receive start: timestamp when the first packet of this frame was received + * 10) Receive finish: timestamp when the last packet of this frame was received + * 11) Decode start: timestamp when the frame was passed to decoder + * 12) Decode finish: timestamp when the frame was decoded + * 13) Render time: timestamp of the projected render time for this frame + * 14) "is outlier": a flag for if this frame is bigger in encoded size than the average frame by at least 5x. + * 15) "triggered by timer": a flag for if this report was triggered by the timer (The report is sent every 200ms) + */ +export class FrameTimingInfo { + rtpTimestamp: number; + captureTimestamp: number; + encodeStartTimestamp: number; + encodeFinishTimestamp: number; + packetizerFinishTimestamp: number; + pacerExitTimestamp: number; + networkTimestamp1: number; + networkTimestamp2: number; + receiveStart: number; + receiveFinish: number; + decodeStart: number; + decodeFinish: number; + renderTime: number; + isOutlier: boolean; + isTriggeredByTimer: boolean; + + /* Milliseconds between encoder start and finish */ + encoderLatencyMs: number; + + /* Milliseconds between encode end and packetizer finish time */ + packetizeLatencyMs: number; + + /* Milliseconds between packetize finish time and pacer sending the frame */ + pacerLatencyMs: number; + + /* Milliseconds between capture time and pacer exit */ + captureToSendLatencyMs: number; +} + /** * Calculates a combination of latency statistics using purely WebRTC API. */ export class LatencyCalculator { + /* Clock offset between peer clocks cannot always be calculated as it relies of latest sender reports. + * so we store the last time we had a valid clock offset in the assumption that clocks haven't drifted too much since then. + */ + private latestSenderRecvClockOffset: number | null = null; + public calculate(stats: AggregatedStats, receivers: RTCRtpReceiver[]): LatencyInfo { const latencyInfo = new LatencyInfo(); - const activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); + const activeCandidatePair: CandidatePairStats | null = stats.getActiveCandidatePair(); if ( !!activeCandidatePair && @@ -32,12 +88,15 @@ export class LatencyCalculator { activeCandidatePair.currentRoundTripTime > 0 ) { // Get RTT - latencyInfo.RTTMs = activeCandidatePair.currentRoundTripTime * 1000; + latencyInfo.rttMs = activeCandidatePair.currentRoundTripTime * 1000; // Calculate sender latency using the first valid video ssrc/csrc - const captureSource: RTCRtpCaptureSource = this.getCaptureSource(receivers); - if (captureSource !== null) { - latencyInfo.SenderLatencyMs = this.calculateSenderLatency(stats, captureSource); + const captureSource: RTCRtpCaptureSource | null = this.getCaptureSource(receivers); + if (captureSource != null) { + const senderLatencyMs = this.calculateSenderLatency(stats, captureSource); + if (senderLatencyMs !== null) { + latencyInfo.senderLatencyMs = senderLatencyMs; + } } } @@ -46,7 +105,7 @@ export class LatencyCalculator { stats.inboundVideoStats.totalProcessingDelay !== undefined && stats.inboundVideoStats.framesDecoded !== undefined ) { - latencyInfo.AverageProcessingDelayMs = + latencyInfo.averageProcessingDelayMs = (stats.inboundVideoStats.totalProcessingDelay / stats.inboundVideoStats.framesDecoded) * 1000; } @@ -55,7 +114,7 @@ export class LatencyCalculator { stats.inboundVideoStats.jitterBufferDelay !== undefined && stats.inboundVideoStats.jitterBufferEmittedCount !== undefined ) { - latencyInfo.AverageJitterBufferDelayMs = + latencyInfo.averageJitterBufferDelayMs = (stats.inboundVideoStats.jitterBufferDelay / stats.inboundVideoStats.jitterBufferEmittedCount) * 1000; @@ -66,7 +125,7 @@ export class LatencyCalculator { stats.inboundVideoStats.framesDecoded !== undefined && stats.inboundVideoStats.totalDecodeTime !== undefined ) { - latencyInfo.AverageDecodeLatencyMs = + latencyInfo.averageDecodeLatencyMs = (stats.inboundVideoStats.totalDecodeTime / stats.inboundVideoStats.framesDecoded) * 1000; } @@ -75,36 +134,102 @@ export class LatencyCalculator { stats.inboundVideoStats.totalAssemblyTime !== undefined && stats.inboundVideoStats.framesAssembledFromMultiplePackets !== undefined ) { - latencyInfo.AverageAssemblyDelayMs = + latencyInfo.averageAssemblyDelayMs = (stats.inboundVideoStats.totalAssemblyTime / stats.inboundVideoStats.framesAssembledFromMultiplePackets) * 1000; } + // Extract extra Chrome-specific stats like encoding latency + if ( + stats.inboundVideoStats.googTimingFrameInfo !== undefined && + stats.inboundVideoStats.googTimingFrameInfo.length > 0 + ) { + latencyInfo.frameTiming = this.extractFrameTimingInfo( + stats.inboundVideoStats.googTimingFrameInfo + ); + } + + // If we could not calculate latency because `senderLatencyMs` was missing because of no SenderReport yet + // We can try to substitute frame timing capture to send latency instead. + if ( + latencyInfo.senderLatencyMs === undefined && + latencyInfo.frameTiming !== undefined && + latencyInfo.frameTiming.captureToSendLatencyMs !== undefined + ) { + latencyInfo.senderLatencyMs = latencyInfo.frameTiming.captureToSendLatencyMs; + } + // Calculate E2E latency as sender-side latency + network latency + receiver-side latency if ( - latencyInfo.AverageProcessingDelayMs !== undefined && - latencyInfo.AverageProcessingDelayMs > 0 && - latencyInfo.SenderLatencyMs != undefined && - latencyInfo.SenderLatencyMs > 0 && - latencyInfo.RTTMs != undefined && - latencyInfo.RTTMs > 0 + latencyInfo.averageProcessingDelayMs !== undefined && + latencyInfo.senderLatencyMs != undefined && + latencyInfo.rttMs !== undefined ) { - latencyInfo.AverageE2ELatency = - latencyInfo.SenderLatencyMs + latencyInfo.RTTMs * 0.5 + latencyInfo.AverageProcessingDelayMs; + latencyInfo.averageE2ELatency = + latencyInfo.senderLatencyMs + latencyInfo.rttMs * 0.5 + latencyInfo.averageProcessingDelayMs; } return latencyInfo; } - private calculateSenderLatency(stats: AggregatedStats, captureSource: RTCRtpCaptureSource): number { + private extractFrameTimingInfo(googTimingFrameInfo: string): FrameTimingInfo { + const timingInfo: FrameTimingInfo = new FrameTimingInfo(); + + const timingInfoArr: string[] = googTimingFrameInfo.split(','); + + // Should have exactly 15 elements according to: + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/video/video_timing.cc;l=82;drc=8d399817282e3c12ed54eb23ec42a5e418298ec6 + if (timingInfoArr.length === 15) { + timingInfo.rtpTimestamp = Number.parseInt(timingInfoArr[0]); + timingInfo.captureTimestamp = Number.parseInt(timingInfoArr[1]); + timingInfo.encodeStartTimestamp = Number.parseInt(timingInfoArr[2]); + timingInfo.encodeFinishTimestamp = Number.parseInt(timingInfoArr[3]); + timingInfo.packetizerFinishTimestamp = Number.parseInt(timingInfoArr[4]); + timingInfo.pacerExitTimestamp = Number.parseInt(timingInfoArr[5]); + timingInfo.networkTimestamp1 = Number.parseInt(timingInfoArr[6]); + timingInfo.networkTimestamp2 = Number.parseInt(timingInfoArr[7]); + timingInfo.receiveStart = Number.parseInt(timingInfoArr[8]); + timingInfo.receiveFinish = Number.parseInt(timingInfoArr[9]); + timingInfo.decodeStart = Number.parseInt(timingInfoArr[10]); + timingInfo.decodeFinish = Number.parseInt(timingInfoArr[11]); + timingInfo.renderTime = Number.parseInt(timingInfoArr[12]); + timingInfo.isOutlier = Number.parseInt(timingInfoArr[13]) > 0; + timingInfo.isTriggeredByTimer = Number.parseInt(timingInfoArr[14]) > 0; + + // Calculate some latency stats + timingInfo.encoderLatencyMs = timingInfo.encodeFinishTimestamp - timingInfo.encodeStartTimestamp; + timingInfo.packetizeLatencyMs = + timingInfo.packetizerFinishTimestamp - timingInfo.encodeFinishTimestamp; + timingInfo.pacerLatencyMs = timingInfo.pacerExitTimestamp - timingInfo.packetizerFinishTimestamp; + timingInfo.captureToSendLatencyMs = timingInfo.pacerExitTimestamp - timingInfo.captureTimestamp; + } + + return timingInfo; + } + + private calculateSenderLatency( + stats: AggregatedStats, + captureSource: RTCRtpCaptureSource + ): number | null { // The calculation performed in this function is as per the procedure defined here: // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset // Get the sender capture in the sender's clock const senderCaptureTimestamp = captureSource.captureTimestamp + captureSource.senderCaptureTimeOffset; - const sendRecvClockOffset = this.calculateSenderReceiverClockOffset(stats); + let sendRecvClockOffset: number | null = this.calculateSenderReceiverClockOffset(stats); + + // Use latest clock offset if we couldn't calculate one now + if (sendRecvClockOffset == null) { + if (this.latestSenderRecvClockOffset != null) { + sendRecvClockOffset = this.latestSenderRecvClockOffset; + } else { + return null; + } + } else { + this.latestSenderRecvClockOffset = sendRecvClockOffset; + } // This brings sender clock roughly inline with recv clock const recvCaptureTimestampNTP = senderCaptureTimestamp + sendRecvClockOffset; @@ -124,7 +249,7 @@ export class LatencyCalculator { * @param receivers The RTP receviers this peer connection has. * @returns A single valid ssrc or csrc that has capture time fields or null if there is none (e.g. in non-chromium browsers it will be null). */ - private getCaptureSource(receivers: RTCRtpReceiver[]): RTCRtpCaptureSource { + private getCaptureSource(receivers: RTCRtpReceiver[]): RTCRtpCaptureSource | null { // We only want video receivers receivers = receivers.filter((receiver) => receiver.track.kind === 'video'); @@ -154,17 +279,16 @@ export class LatencyCalculator { return null; } - private calculateSenderReceiverClockOffset(stats: AggregatedStats) { + private calculateSenderReceiverClockOffset(stats: AggregatedStats): number | null { // The calculation performed in this function is as per the procedure defined here: // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset const remoteVideoStatsArrivedTimestamp = stats.outBoundVideoStats.timestamp; const remoteVideoStatsSentTimestamp = stats.outBoundVideoStats.remoteTimestamp; - const activeCandidatePair: CandidatePairStats = stats.getActiveCandidatePair(); - const networkDelay = activeCandidatePair - ? activeCandidatePair.currentRoundTripTime * 0.5 * 1000 - : 0.0; + const activeCandidatePair: CandidatePairStats | null = stats.getActiveCandidatePair(); + const networkDelay = + activeCandidatePair != null ? activeCandidatePair.currentRoundTripTime * 0.5 * 1000 : 0.0; if ( remoteVideoStatsArrivedTimestamp !== undefined && @@ -173,9 +297,10 @@ export class LatencyCalculator { ) { return remoteVideoStatsArrivedTimestamp - (remoteVideoStatsSentTimestamp + networkDelay); } - - Logger.Warning('Could not get stats to calculate sender/receiver clock offset.'); - return 0.0; + // Could not get stats to calculate sender/receiver clock offset + else { + return null; + } } } @@ -190,23 +315,26 @@ export class LatencyInfo { * Note: This can only be calculated if both offer and answer contain the * the RTP header extension for `abs-capture-time`. */ - public SenderLatencyMs: number | undefined = undefined; + public senderLatencyMs: number | undefined = undefined; /* The round trip time (milliseconds) between each sender->receiver->sender */ - public RTTMs: number | undefined = undefined; + public rttMs: number | undefined = undefined; /* Average time taken (milliseconds) from video packet receipt to post-decode. */ - public AverageProcessingDelayMs: number | undefined = undefined; + public averageProcessingDelayMs: number | undefined = undefined; /* Average time taken (milliseconds) inside the jitter buffer (which is post-receipt but pre-decode). */ - public AverageJitterBufferDelayMs: number | undefined = undefined; + public averageJitterBufferDelayMs: number | undefined = undefined; /* Average time taken (milliseconds) to decode a video frame. */ - public AverageDecodeLatencyMs: number | undefined = undefined; + public averageDecodeLatencyMs: number | undefined = undefined; /* Average time taken (milliseconds) to between receipt of the first and last video packet of a. */ - public AverageAssemblyDelayMs: number | undefined = undefined; + public averageAssemblyDelayMs: number | undefined = undefined; /* The sender latency + RTT/2 + processing delay */ - public AverageE2ELatency: number | undefined = undefined; + public averageE2ELatency: number | undefined = undefined; + + /* Timing information about the worst performing frame since the last getStats call (only works on Chrome) */ + public frameTiming: FrameTimingInfo | undefined = undefined; } diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 921582c53..cb2159b49 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -171,18 +171,9 @@ export class PeerConnectionController { * Generate Aggregated Stats and then fire a onVideo Stats event */ generateStats() { - const audioPromise = this.audioTrack - ? this.peerConnection?.getStats(this.audioTrack).then((statsData: RTCStatsReport) => { - this.aggregatedStats.processStats(statsData); - }) - : Promise.resolve(); - const videoPromise = this.videoTrack - ? this.peerConnection?.getStats(this.videoTrack).then((statsData: RTCStatsReport) => { - this.aggregatedStats.processStats(statsData); - }) - : Promise.resolve(); - - Promise.allSettled([audioPromise, videoPromise]).then(() => { + this.peerConnection.getStats().then((statsData: RTCStatsReport) => { + this.aggregatedStats.processStats(statsData); + this.onVideoStats(this.aggregatedStats); // Calculate latency using stats and video receivers and then call the handling function @@ -195,9 +186,14 @@ export class PeerConnectionController { // Update the preferred codec selection based on what was actually negotiated if (this.updateCodecSelection && !!this.aggregatedStats.inboundVideoStats.codecId) { // Construct the qualified codec name from the mimetype and fmtp - const codecStats: CodecStats = this.aggregatedStats.codecs.get( + const codecStats: CodecStats | undefined = this.aggregatedStats.codecs.get( this.aggregatedStats.inboundVideoStats.codecId ); + + if (codecStats === undefined) { + return; + } + const codecShortname = codecStats.mimeType.replace('video/', ''); let fullCodecName = codecShortname; if (codecStats.sdpFmtpLine && codecStats.sdpFmtpLine.trim() !== '') { diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index 7e6426f92..b3496e685 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -299,18 +299,17 @@ export class StatsPanel { } // Store the active candidate pair return a new Candidate pair stat if getActiveCandidate is null - const activeCandidatePair = - stats.getActiveCandidatePair() != null - ? stats.getActiveCandidatePair() - : new CandidatePairStats(); - - // RTT - const netRTT = - Object.prototype.hasOwnProperty.call(activeCandidatePair, 'currentRoundTripTime') && - stats.isNumber(activeCandidatePair.currentRoundTripTime) - ? Math.ceil(activeCandidatePair.currentRoundTripTime * 1000).toString() - : "Can't calculate"; - this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT); + const activeCandidatePair: CandidatePairStats | null = stats.getActiveCandidatePair(); + + if (activeCandidatePair) { + // RTT + const netRTT = + Object.prototype.hasOwnProperty.call(activeCandidatePair, 'currentRoundTripTime') && + stats.isNumber(activeCandidatePair.currentRoundTripTime) + ? Math.ceil(activeCandidatePair.currentRoundTripTime * 1000).toString() + : "Can't calculate"; + this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT); + } this.addOrUpdateStat('DurationStat', 'Duration', stats.sessionStats.runTime); @@ -336,54 +335,89 @@ export class StatsPanel { } public handleLatencyInfo(latencyInfo: LatencyInfo) { - if (latencyInfo.SenderLatencyMs !== undefined && latencyInfo.SenderLatencyMs > 0) { + if (latencyInfo.frameTiming !== undefined) { + // Encoder latency + if (latencyInfo.frameTiming.encoderLatencyMs !== undefined) { + this.addOrUpdateStat( + 'EncodeLatency', + 'Encode latency (ms)', + Math.ceil(latencyInfo.frameTiming.encoderLatencyMs).toString() + ); + } + + // Packetizer latency + if (latencyInfo.frameTiming.packetizeLatencyMs !== undefined) { + this.addOrUpdateStat( + 'PacketizerLatency', + 'Packetizer latency (ms)', + Math.ceil(latencyInfo.frameTiming.packetizeLatencyMs).toString() + ); + } + + // Pacer latency + if (latencyInfo.frameTiming.pacerLatencyMs !== undefined) { + this.addOrUpdateStat( + 'PacerLatency', + 'Pacer latency (ms)', + Math.ceil(latencyInfo.frameTiming.pacerLatencyMs).toString() + ); + } + + // Sender latency calculated using timing stats + if (latencyInfo.frameTiming.captureToSendLatencyMs !== undefined) { + this.addOrUpdateStat( + 'CaptureToSend', + 'Capture to send latency (ms)', + Math.ceil(latencyInfo.frameTiming.captureToSendLatencyMs).toString() + ); + } + } + + if (latencyInfo.senderLatencyMs !== undefined) { this.addOrUpdateStat( 'SenderSideLatency', 'Sender latency (ms)', - Math.ceil(latencyInfo.SenderLatencyMs).toString() + Math.ceil(latencyInfo.senderLatencyMs).toString() ); } - if (latencyInfo.AverageAssemblyDelayMs !== undefined && latencyInfo.AverageAssemblyDelayMs > 0) { + if (latencyInfo.averageAssemblyDelayMs !== undefined) { this.addOrUpdateStat( 'AvgAssemblyDelay', 'Assembly delay (ms)', - Math.ceil(latencyInfo.AverageAssemblyDelayMs).toString() + Math.ceil(latencyInfo.averageAssemblyDelayMs).toString() ); } - if (latencyInfo.AverageDecodeLatencyMs !== undefined && latencyInfo.AverageDecodeLatencyMs > 0) { + if (latencyInfo.averageDecodeLatencyMs !== undefined) { this.addOrUpdateStat( 'AvgDecodeDelay', 'Decode time (ms)', - Math.ceil(latencyInfo.AverageDecodeLatencyMs).toString() + Math.ceil(latencyInfo.averageDecodeLatencyMs).toString() ); } - if ( - latencyInfo.AverageJitterBufferDelayMs !== undefined && - latencyInfo.AverageJitterBufferDelayMs > 0 - ) { + if (latencyInfo.averageJitterBufferDelayMs !== undefined) { this.addOrUpdateStat( 'AvgJitterBufferDelay', 'Jitter buffer (ms)', - Math.ceil(latencyInfo.AverageJitterBufferDelayMs).toString() + Math.ceil(latencyInfo.averageJitterBufferDelayMs).toString() ); } - if (latencyInfo.AverageProcessingDelayMs !== undefined && latencyInfo.AverageProcessingDelayMs > 0) { + if (latencyInfo.averageProcessingDelayMs !== undefined) { this.addOrUpdateStat( 'AvgProcessingDelay', 'Processing delay (ms)', - Math.ceil(latencyInfo.AverageProcessingDelayMs).toString() + Math.ceil(latencyInfo.averageProcessingDelayMs).toString() ); } - if (latencyInfo.AverageE2ELatency !== undefined && latencyInfo.AverageE2ELatency > 0) { + if (latencyInfo.averageE2ELatency !== undefined) { this.addOrUpdateStat( 'AvgE2ELatency', 'Total latency (ms)', - Math.ceil(latencyInfo.AverageE2ELatency).toString() + Math.ceil(latencyInfo.averageE2ELatency).toString() ); } } From 701c265e42f30b68a029f33edd8ed59630ad0ffa Mon Sep 17 00:00:00 2001 From: Luke Bermingham Date: Fri, 7 Feb 2025 10:58:50 +1000 Subject: [PATCH 24/27] Tided up how latency stats are displayed in stats panel --- .../tests/peerconnection.spec.ts | 2 +- Extras/JSStreamer/src/streamer.ts | 6 +- .../AggregatedStats.ts | 135 +++++++++----- .../InboundRTPStats.ts | 173 +++++++++--------- .../LatencyCalculator.ts | 78 ++++++-- .../OutBoundRTPStats.ts | 58 ++++-- .../library/src/pixelstreamingfrontend.ts | 2 +- Frontend/ui-library/src/UI/StatsPanel.ts | 170 ++++++++++++----- .../ui-library/src/UI/UIConfigurationTypes.ts | 1 + 9 files changed, 418 insertions(+), 207 deletions(-) diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 0796b713a..41459fe72 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -120,7 +120,7 @@ test('Test latency calculation', { let latencyInfo: LatencyInfo = await page.evaluate(() => { return new Promise((resolve) => { window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => { - if(e.data.latencyInfo && e.data.latencyInfo.senderLatencyMs) { + if(e.data.latencyInfo && e.data.latencyInfo.senderLatencyMs && e.data.latencyInfo.rttMs) { resolve(e.data.latencyInfo); } }); diff --git a/Extras/JSStreamer/src/streamer.ts b/Extras/JSStreamer/src/streamer.ts index fc8003544..08ef1fd05 100644 --- a/Extras/JSStreamer/src/streamer.ts +++ b/Extras/JSStreamer/src/streamer.ts @@ -184,7 +184,7 @@ export class Streamer extends EventEmitter { } const tranceiverOptions: RTCRtpTransceiverInit = { streams: [this.localStream], - direction: 'sendonly', + direction: 'sendrecv', sendEncodings: [ { maxBitrate: this.settings.WebRTC.MaxBitrate, @@ -224,8 +224,10 @@ export class Streamer extends EventEmitter { dataChannel: dataChannel }; + const offerOptions: RTCOfferOptions = { offerToReceiveAudio: true, offerToReceiveVideo: true } + peerConnection - .createOffer() + .createOffer(offerOptions) .then((offer) => { if(offer.sdp == undefined) { diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 28af72cc9..a4a0c5ebc 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -5,7 +5,7 @@ import { InboundTrackStats } from './InboundTrackStats'; import { DataChannelStats } from './DataChannelStats'; import { CandidateStat } from './CandidateStat'; import { CandidatePairStats } from './CandidatePairStats'; -import { OutBoundRTPStats, OutBoundVideoStats } from './OutBoundRTPStats'; +import { RemoteOutboundRTPStats, OutboundRTPStats } from './OutBoundRTPStats'; import { SessionStats } from './SessionStats'; import { StreamStats } from './StreamStats'; import { CodecStats } from './CodecStats'; @@ -19,10 +19,13 @@ export class AggregatedStats { inboundVideoStats: InboundVideoStats; inboundAudioStats: InboundAudioStats; candidatePairs: Array; - DataChannelStats: DataChannelStats; + datachannelStats: DataChannelStats; localCandidates: Array; remoteCandidates: Array; - outBoundVideoStats: OutBoundVideoStats; + outboundVideoStats: OutboundRTPStats; + outboundAudioStats: OutboundRTPStats; + remoteOutboundVideoStats: RemoteOutboundRTPStats; + remoteOutboundAudioStats: RemoteOutboundRTPStats; sessionStats: SessionStats; streamStats: StreamStats; codecs: Map; @@ -31,8 +34,11 @@ export class AggregatedStats { constructor() { this.inboundVideoStats = new InboundVideoStats(); this.inboundAudioStats = new InboundAudioStats(); - this.DataChannelStats = new DataChannelStats(); - this.outBoundVideoStats = new OutBoundVideoStats(); + this.datachannelStats = new DataChannelStats(); + this.outboundVideoStats = new OutboundRTPStats(); + this.outboundAudioStats = new OutboundRTPStats(); + this.remoteOutboundAudioStats = new RemoteOutboundRTPStats(); + this.remoteOutboundVideoStats = new RemoteOutboundRTPStats(); this.sessionStats = new SessionStats(); this.streamStats = new StreamStats(); this.codecs = new Map(); @@ -63,7 +69,7 @@ export class AggregatedStats { this.handleDataChannel(stat); break; case 'inbound-rtp': - this.handleInBoundRTP(stat); + this.handleInboundRTP(stat); break; case 'local-candidate': this.handleLocalCandidate(stat); @@ -73,6 +79,7 @@ export class AggregatedStats { case 'media-playout': break; case 'outbound-rtp': + this.handleLocalOutbound(stat); break; case 'peer-connection': break; @@ -82,7 +89,7 @@ export class AggregatedStats { case 'remote-inbound-rtp': break; case 'remote-outbound-rtp': - this.handleRemoteOutBound(stat); + this.handleRemoteOutbound(stat); break; case 'track': this.handleTrack(stat); @@ -124,16 +131,16 @@ export class AggregatedStats { * @param stat - the stats coming in from the data channel */ handleDataChannel(stat: DataChannelStats) { - this.DataChannelStats.bytesReceived = stat.bytesReceived; - this.DataChannelStats.bytesSent = stat.bytesSent; - this.DataChannelStats.dataChannelIdentifier = stat.dataChannelIdentifier; - this.DataChannelStats.id = stat.id; - this.DataChannelStats.label = stat.label; - this.DataChannelStats.messagesReceived = stat.messagesReceived; - this.DataChannelStats.messagesSent = stat.messagesSent; - this.DataChannelStats.protocol = stat.protocol; - this.DataChannelStats.state = stat.state; - this.DataChannelStats.timestamp = stat.timestamp; + this.datachannelStats.bytesReceived = stat.bytesReceived; + this.datachannelStats.bytesSent = stat.bytesSent; + this.datachannelStats.dataChannelIdentifier = stat.dataChannelIdentifier; + this.datachannelStats.id = stat.id; + this.datachannelStats.label = stat.label; + this.datachannelStats.messagesReceived = stat.messagesReceived; + this.datachannelStats.messagesSent = stat.messagesSent; + this.datachannelStats.protocol = stat.protocol; + this.datachannelStats.state = stat.state; + this.datachannelStats.timestamp = stat.timestamp; } /** @@ -158,23 +165,23 @@ export class AggregatedStats { * @param stat - ice candidate stats */ handleRemoteCandidate(stat: CandidateStat) { - const RemoteCandidate = new CandidateStat(); - RemoteCandidate.label = 'remote-candidate'; - RemoteCandidate.address = stat.address; - RemoteCandidate.port = stat.port; - RemoteCandidate.protocol = stat.protocol; - RemoteCandidate.id = stat.id; - RemoteCandidate.candidateType = stat.candidateType; - RemoteCandidate.relayProtocol = stat.relayProtocol; - RemoteCandidate.transportId = stat.transportId; - this.remoteCandidates.push(RemoteCandidate); + const remoteCandidate = new CandidateStat(); + remoteCandidate.label = 'remote-candidate'; + remoteCandidate.address = stat.address; + remoteCandidate.port = stat.port; + remoteCandidate.protocol = stat.protocol; + remoteCandidate.id = stat.id; + remoteCandidate.candidateType = stat.candidateType; + remoteCandidate.relayProtocol = stat.relayProtocol; + remoteCandidate.transportId = stat.transportId; + this.remoteCandidates.push(remoteCandidate); } /** * Process the Inbound RTP Audio and Video Data * @param stat - inbound rtp stats */ - handleInBoundRTP(stat: InboundRTPStats) { + handleInboundRTP(stat: InboundRTPStats) { switch (stat.kind) { case 'video': // Calculate bitrate between stat updates @@ -215,25 +222,63 @@ export class AggregatedStats { } /** - * Process the outbound RTP Audio and Video Data - * @param stat - remote outbound stats + * Process the "local" outbound RTP Audio and Video stats. + * @param stat - local outbound rtp stats */ - handleRemoteOutBound(stat: OutBoundRTPStats) { - switch (stat.kind) { - case 'video': - this.outBoundVideoStats.bytesSent = stat.bytesSent; - this.outBoundVideoStats.id = stat.id; - this.outBoundVideoStats.localId = stat.localId; - this.outBoundVideoStats.packetsSent = stat.packetsSent; - this.outBoundVideoStats.remoteTimestamp = stat.remoteTimestamp; - this.outBoundVideoStats.timestamp = stat.timestamp; - break; - case 'audio': - break; + handleLocalOutbound(stat: OutboundRTPStats) { + const localOutboundStats: OutboundRTPStats = + stat.kind === 'audio' ? this.outboundAudioStats : this.outboundVideoStats; + localOutboundStats.active = stat.active; + localOutboundStats.codecId = stat.codecId; + localOutboundStats.bytesSent = stat.bytesSent; + localOutboundStats.frameHeight = stat.frameHeight; + localOutboundStats.frameWidth = stat.frameWidth; + localOutboundStats.framesEncoded = stat.framesEncoded; + localOutboundStats.framesPerSecond = stat.framesPerSecond; + localOutboundStats.headerBytesSent = stat.headerBytesSent; + localOutboundStats.id = stat.id; + localOutboundStats.keyFramesEncoded = stat.keyFramesEncoded; + localOutboundStats.kind = stat.kind; + localOutboundStats.mediaSourceId = stat.mediaSourceId; + localOutboundStats.mid = stat.mid; + localOutboundStats.nackCount = stat.nackCount; + localOutboundStats.packetsSent = stat.packetsSent; + localOutboundStats.qpSum = stat.qpSum; + localOutboundStats.qualityLimitationDurations = stat.qualityLimitationDurations; + localOutboundStats.qualityLimitationReason = stat.qualityLimitationReason; + localOutboundStats.remoteId = stat.remoteId; + localOutboundStats.retransmittedBytesSent = stat.retransmittedBytesSent; + localOutboundStats.rid = stat.rid; + localOutboundStats.scalabilityMode = stat.scalabilityMode; + localOutboundStats.ssrc = stat.ssrc; + localOutboundStats.targetBitrate = stat.targetBitrate; + localOutboundStats.timestamp = stat.timestamp; + localOutboundStats.totalEncodeTime = stat.totalEncodeTime; + localOutboundStats.totalEncodeBytesTarget = stat.totalEncodeBytesTarget; + localOutboundStats.totalPacketSendDelay = stat.totalPacketSendDelay; + localOutboundStats.transportId = stat.transportId; + } - default: - break; - } + /** + * Process the "remote" outbound RTP Audio and Video stats. + * @param stat - remote outbound rtp stats + */ + handleRemoteOutbound(stat: RemoteOutboundRTPStats) { + const remoteOutboundStats: RemoteOutboundRTPStats = + stat.kind === 'audio' ? this.remoteOutboundAudioStats : this.remoteOutboundVideoStats; + remoteOutboundStats.bytesSent = stat.bytesSent; + remoteOutboundStats.codecId = stat.codecId; + remoteOutboundStats.id = stat.id; + remoteOutboundStats.kind = stat.kind; + remoteOutboundStats.localId = stat.localId; + remoteOutboundStats.packetsSent = stat.packetsSent; + remoteOutboundStats.remoteTimestamp = stat.remoteTimestamp; + remoteOutboundStats.reportsSent = stat.reportsSent; + remoteOutboundStats.roundTripTimeMeasurements = stat.roundTripTimeMeasurements; + remoteOutboundStats.ssrc = stat.ssrc; + remoteOutboundStats.timestamp = stat.timestamp; + remoteOutboundStats.totalRoundTripTime = stat.totalRoundTripTime; + remoteOutboundStats.transportId = stat.transportId; } /** diff --git a/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts b/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts index 030b9d43b..663c6c5e7 100644 --- a/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts +++ b/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts @@ -4,41 +4,41 @@ * Inbound Audio Stats collected from the RTC Stats Report */ export class InboundAudioStats { - audioLevel: number; + audioLevel: number | undefined; bytesReceived: number; codecId: string; - concealedSamples: number; - concealmentEvents: number; - fecPacketsDiscarded: number; - fecPacketsReceived: number; + concealedSamples: number | undefined; + concealmentEvents: number | undefined; + fecPacketsDiscarded: number | undefined; + fecPacketsReceived: number | undefined; headerBytesReceived: number; id: string; - insertedSamplesForDeceleration: number; + insertedSamplesForDeceleration: number | undefined; jitter: number; jitterBufferDelay: number; jitterBufferEmittedCount: number; - jitterBufferMinimumDelay: number; - jitterBufferTargetDelay: number; + jitterBufferMinimumDelay: number | undefined; + jitterBufferTargetDelay: number | undefined; kind: string; lastPacketReceivedTimestamp: number; - mediaType: string; + mediaType: string | undefined; mid: string; - packetsDiscarded: number; + packetsDiscarded: number | undefined; packetsLost: number; packetsReceived: number; - removedSamplesForAcceleration: number; - silentConcealedSamples: number; + removedSamplesForAcceleration: number | undefined; + silentConcealedSamples: number | undefined; ssrc: number; timestamp: number; - totalAudioEnergy: number; - totalSamplesDuration: number; - totalSamplesReceived: number; - trackIdentifier: string; - transportId: string; + totalAudioEnergy: number | undefined; + totalSamplesDuration: number | undefined; + totalSamplesReceived: number | undefined; + trackIdentifier: string | undefined; + transportId: string | undefined; type: string; /* additional, custom stats */ - bitrate: number; + bitrate: number | undefined; } /** @@ -46,47 +46,47 @@ export class InboundAudioStats { */ export class InboundVideoStats { bytesReceived: number; - codecId: string; - firCount: number; - frameHeight: number; - frameWidth: number; - framesAssembledFromMultiplePackets: number; - framesDecoded: number; - framesDropped: number; - framesPerSecond: number; - framesReceived: number; - freezeCount: number; - googTimingFrameInfo: string; + codecId: string | undefined; + firCount: number | undefined; + frameHeight: number | undefined; + frameWidth: number | undefined; + framesAssembledFromMultiplePackets: number | undefined; + framesDecoded: number | undefined; + framesDropped: number | undefined; + framesPerSecond: number | undefined; + framesReceived: number | undefined; + freezeCount: number | undefined; + googTimingFrameInfo: string | undefined; headerBytesReceived: number; id: string; jitter: number; jitterBufferDelay: number; jitterBufferEmittedCount: number; - keyFramesDecoded: number; + keyFramesDecoded: number | undefined; kind: string; - lastPacketReceivedTimestamp: number; - mediaType: string; + lastPacketReceivedTimestamp: number | undefined; + mediaType: string | undefined; mid: string; - nackCount: number; + nackCount: number | undefined; packetsLost: number; packetsReceived: number; - pauseCount: number; - pliCount: number; + pauseCount: number | undefined; + pliCount: number | undefined; ssrc: number; timestamp: number; - totalAssemblyTime: number; - totalDecodeTime: number; - totalFreezesDuration: number; - totalInterFrameDelay: number; - totalPausesDuration: number; - totalProcessingDelay: number; - totalSquaredInterFrameDelay: number; - trackIdentifier: string; - transportId: string; + totalAssemblyTime: number | undefined; + totalDecodeTime: number | undefined; + totalFreezesDuration: number | undefined; + totalInterFrameDelay: number | undefined; + totalPausesDuration: number | undefined; + totalProcessingDelay: number | undefined; + totalSquaredInterFrameDelay: number | undefined; + trackIdentifier: string | undefined; + transportId: string | undefined; type: string; /* additional, custom stats */ - bitrate: number; + bitrate: number | undefined; } /** @@ -95,60 +95,63 @@ export class InboundVideoStats { export class InboundRTPStats { /* common stats */ bytesReceived: number; - codecId: string; + codecId: string | undefined; headerBytesReceived: number; id: string; jitter: number; jitterBufferDelay: number; jitterBufferEmittedCount: number; kind: string; - lastPacketReceivedTimestamp: number; - mediaType: string; + lastPacketReceivedTimestamp: number | undefined; + mediaType: string | undefined; mid: string; packetsLost: number; packetsReceived: number; + playoutId: string | undefined; + qpsum: number | undefined; + remoteId: string | undefined; ssrc: number; timestamp: number; - trackIdentifier: string; - transportId: string; + trackIdentifier: string | undefined; + transportId: string | undefined; type: string; /* audio specific stats */ - audioLevel: number; - concealedSamples: number; - concealmentEvents: number; - fecPacketsDiscarded: number; - fecPacketsReceived: number; - insertedSamplesForDeceleration: number; - jitterBufferMinimumDelay: number; - jitterBufferTargetDelay: number; - packetsDiscarded: number; - removedSamplesForAcceleration: number; - silentConcealedSamples: number; - totalAudioEnergy: number; - totalSamplesDuration: number; - totalSamplesReceived: number; + audioLevel: number | undefined; + concealedSamples: number | undefined; + concealmentEvents: number | undefined; + fecPacketsDiscarded: number | undefined; + fecPacketsReceived: number | undefined; + insertedSamplesForDeceleration: number | undefined; + jitterBufferMinimumDelay: number | undefined; + jitterBufferTargetDelay: number | undefined; + packetsDiscarded: number | undefined; + removedSamplesForAcceleration: number | undefined; + silentConcealedSamples: number | undefined; + totalAudioEnergy: number | undefined; + totalSamplesDuration: number | undefined; + totalSamplesReceived: number | undefined; /* video specific stats */ - firCount: number; - frameHeight: number; - frameWidth: number; - framesAssembledFromMultiplePackets: number; - framesDecoded: number; - framesDropped: number; - framesPerSecond: number; - framesReceived: number; - freezeCount: number; - googTimingFrameInfo: string; - keyFramesDecoded: number; - nackCount: number; - pauseCount: number; - pliCount: number; - totalAssemblyTime: number; - totalDecodeTime: number; - totalFreezesDuration: number; - totalInterFrameDelay: number; - totalPausesDuration: number; - totalProcessingDelay: number; - totalSquaredInterFrameDelay: number; + firCount: number | undefined; + frameHeight: number | undefined; + frameWidth: number | undefined; + framesAssembledFromMultiplePackets: number | undefined; + framesDecoded: number | undefined; + framesDropped: number | undefined; + framesPerSecond: number | undefined; + framesReceived: number | undefined; + freezeCount: number | undefined; + googTimingFrameInfo: string | undefined; + keyFramesDecoded: number | undefined; + nackCount: number | undefined; + pauseCount: number | undefined; + pliCount: number | undefined; + totalAssemblyTime: number | undefined; + totalDecodeTime: number | undefined; + totalFreezesDuration: number | undefined; + totalInterFrameDelay: number | undefined; + totalPausesDuration: number | undefined; + totalProcessingDelay: number | undefined; + totalSquaredInterFrameDelay: number | undefined; } diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts index 2d10694f1..c67291eb1 100644 --- a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -80,15 +80,10 @@ export class LatencyCalculator { public calculate(stats: AggregatedStats, receivers: RTCRtpReceiver[]): LatencyInfo { const latencyInfo = new LatencyInfo(); - const activeCandidatePair: CandidatePairStats | null = stats.getActiveCandidatePair(); + const rttMS: number | null = this.getRTTMs(stats); - if ( - !!activeCandidatePair && - activeCandidatePair.currentRoundTripTime !== undefined && - activeCandidatePair.currentRoundTripTime > 0 - ) { - // Get RTT - latencyInfo.rttMs = activeCandidatePair.currentRoundTripTime * 1000; + if (rttMS != null) { + latencyInfo.rttMs = rttMS; // Calculate sender latency using the first valid video ssrc/csrc const captureSource: RTCRtpCaptureSource | null = this.getCaptureSource(receivers); @@ -160,7 +155,7 @@ export class LatencyCalculator { latencyInfo.senderLatencyMs = latencyInfo.frameTiming.captureToSendLatencyMs; } - // Calculate E2E latency as sender-side latency + network latency + receiver-side latency + // Calculate E2E latency as sender-side latency + one way network latency + receiver-side latency if ( latencyInfo.averageProcessingDelayMs !== undefined && latencyInfo.senderLatencyMs != undefined && @@ -283,25 +278,72 @@ export class LatencyCalculator { // The calculation performed in this function is as per the procedure defined here: // https://w3c.github.io/webrtc-extensions/#dom-rtcrtpcontributingsource-sendercapturetimeoffset - const remoteVideoStatsArrivedTimestamp = stats.outBoundVideoStats.timestamp; - const remoteVideoStatsSentTimestamp = stats.outBoundVideoStats.remoteTimestamp; + const hasRemoteOutboundVideoStats = + stats.remoteOutboundVideoStats !== undefined && + stats.remoteOutboundVideoStats.timestamp !== undefined && + stats.remoteOutboundVideoStats.remoteTimestamp !== undefined; - const activeCandidatePair: CandidatePairStats | null = stats.getActiveCandidatePair(); - const networkDelay = - activeCandidatePair != null ? activeCandidatePair.currentRoundTripTime * 0.5 * 1000 : 0.0; + // Note: As of Chrome 132, remote-outbound-rtp stats for video are not yet implemented (audio works). + // This codepath should activate once they do begin to work. + if (!hasRemoteOutboundVideoStats) { + return null; + } + + const remoteStatsArrivedTimestamp = stats.remoteOutboundVideoStats.timestamp; + const remoteStatsSentTimestamp = stats.remoteOutboundVideoStats.remoteTimestamp; + + const rttMs: number | null = this.getRTTMs(stats); if ( - remoteVideoStatsArrivedTimestamp !== undefined && - remoteVideoStatsSentTimestamp !== undefined && - networkDelay !== undefined + remoteStatsArrivedTimestamp !== undefined && + remoteStatsSentTimestamp !== undefined && + rttMs !== null ) { - return remoteVideoStatsArrivedTimestamp - (remoteVideoStatsSentTimestamp + networkDelay); + const onewayDelay = rttMs * 0.5; + return remoteStatsArrivedTimestamp - (remoteStatsSentTimestamp + onewayDelay); } // Could not get stats to calculate sender/receiver clock offset else { return null; } } + + private getRTTMs(stats: AggregatedStats): number | null { + // Try to get it from the active candidate pair + const activeCandidatePair: CandidatePairStats | null = stats.getActiveCandidatePair(); + if (!!activeCandidatePair && activeCandidatePair.currentRoundTripTime !== undefined) { + const curRTTSeconds = activeCandidatePair.currentRoundTripTime; + return curRTTSeconds * 1000; + } + + // Next try to get it from remote-outbound-rtp video stats + if ( + !!stats.remoteOutboundVideoStats && + stats.remoteOutboundVideoStats.totalRoundTripTime !== undefined && + stats.remoteOutboundVideoStats.roundTripTimeMeasurements !== undefined && + stats.remoteOutboundVideoStats.roundTripTimeMeasurements > 0 + ) { + const avgRttSeconds = + stats.remoteOutboundVideoStats.totalRoundTripTime / + stats.remoteOutboundVideoStats.roundTripTimeMeasurements; + return avgRttSeconds * 1000; + } + + // Next try to get it from remote-outbound-rtp audio stats + if ( + !!stats.remoteOutboundAudioStats && + stats.remoteOutboundAudioStats.totalRoundTripTime !== undefined && + stats.remoteOutboundAudioStats.roundTripTimeMeasurements !== undefined && + stats.remoteOutboundAudioStats.roundTripTimeMeasurements > 0 + ) { + const avgRttSeconds = + stats.remoteOutboundAudioStats.totalRoundTripTime / + stats.remoteOutboundAudioStats.roundTripTimeMeasurements; + return avgRttSeconds * 1000; + } + + return null; + } } /** diff --git a/Frontend/library/src/PeerConnectionController/OutBoundRTPStats.ts b/Frontend/library/src/PeerConnectionController/OutBoundRTPStats.ts index 723e9fcc4..f6df02c04 100644 --- a/Frontend/library/src/PeerConnectionController/OutBoundRTPStats.ts +++ b/Frontend/library/src/PeerConnectionController/OutBoundRTPStats.ts @@ -1,26 +1,60 @@ // Copyright Epic Games, Inc. All Rights Reserved. /** - * Outbound Video Stats collected from the RTC Stats Report + * Outbound RTP stats collected from the RTC Stats Report under `outbound-rtp`. + * Wrapper around: https://developer.mozilla.org/en-US/docs/Web/API/RTCOutboundRtpStreamStats + * These are stats for video we are sending to a remote peer. */ -export class OutBoundVideoStats { +export class OutboundRTPStats { + active: boolean | undefined; + codecId: string | undefined; bytesSent: number; + frameHeight: number | undefined; + frameWidth: number | undefined; + framesEncoded: number | undefined; + framesPerSecond: number | undefined; + framesSent: number | undefined; + headerBytesSent: number; id: string; - localId: string; + keyFramesEncoded: number | undefined; + kind: string; + mediaSourceId: string | undefined; + mid: string | undefined; + nackCount: number | undefined; packetsSent: number; - remoteTimestamp: number; + qpSum: number | undefined; + qualityLimitationDurations: number | undefined; + qualityLimitationReason: string | undefined; + remoteId: string | undefined; + retransmittedBytesSent: number; + rid: string | undefined; + scalabilityMode: string | undefined; + ssrc: string; + targetBitrate: number | undefined; timestamp: number; + totalEncodeTime: number | undefined; + totalEncodeBytesTarget: number | undefined; + totalPacketSendDelay: number | undefined; + transportId: string | undefined; } /** - * Outbound Stats collected from the RTC Stats Report + * Remote outbound stats collected from the RTC Stats Report under `remote-outbound-rtp`. + * Wrapper around: https://developer.mozilla.org/en-US/docs/Web/API/RTCRemoteOutboundRtpStreamStats + * These are stats for media we are receiving from a remote peer. */ -export class OutBoundRTPStats { +export class RemoteOutboundRTPStats { + bytesSent: number | undefined; + codecId: string; + id: string | undefined; kind: string; - bytesSent: number; - id: string; - localId: string; - packetsSent: number; - remoteTimestamp: number; - timestamp: number; + localId: string | undefined; + packetsSent: number | undefined; + remoteTimestamp: number | undefined; + reportsSent: number | undefined; + roundTripTimeMeasurements: number | undefined; + ssrc: string; + timestamp: number | undefined; + totalRoundTripTime: number | undefined; + transportId: string | undefined; } diff --git a/Frontend/library/src/pixelstreamingfrontend.ts b/Frontend/library/src/pixelstreamingfrontend.ts index 32a10b150..cc5e1a1bc 100644 --- a/Frontend/library/src/pixelstreamingfrontend.ts +++ b/Frontend/library/src/pixelstreamingfrontend.ts @@ -19,7 +19,7 @@ export { CandidatePairStats } from './PeerConnectionController/CandidatePairStat export { CandidateStat } from './PeerConnectionController/CandidateStat'; export { DataChannelStats } from './PeerConnectionController/DataChannelStats'; export { InboundAudioStats, InboundVideoStats } from './PeerConnectionController/InboundRTPStats'; -export { OutBoundVideoStats } from './PeerConnectionController/OutBoundRTPStats'; +export { OutboundRTPStats, RemoteOutboundRTPStats } from './PeerConnectionController/OutBoundRTPStats'; export * from './PeerConnectionController/LatencyCalculator'; export * from './DataChannel/DataChannelLatencyTestResults'; export * from './Util/EventEmitter'; diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index b3496e685..8521caa63 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -11,7 +11,12 @@ import { import { AggregatedStats } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5'; import { MathUtils } from '../Util/MathUtils'; import { DataChannelLatencyTest } from './DataChannelLatencyTest'; -import { isSectionEnabled, StatsSections, StatsPanelConfiguration } from './UIConfigurationTypes'; +import { + isSectionEnabled, + StatsSections, + StatsSectionsIds, + StatsPanelConfiguration +} from './UIConfigurationTypes'; /** * A stat structure, an id, the stat string, and the element where it is rendered. @@ -31,7 +36,9 @@ export class StatsPanel { _statsCloseButton: HTMLElement; _statsContentElement: HTMLElement; _statisticsContainer: HTMLElement; + _latencyStatsContainer: HTMLElement; _statsResult: HTMLElement; + _latencyResult: HTMLElement; _config: StatsPanelConfiguration; latencyTest: LatencyTest; @@ -88,22 +95,42 @@ export class StatsPanel { statistics.id = 'statistics'; statistics.classList.add('settingsContainer'); + const latencyStats = document.createElement('section'); + latencyStats.id = 'latencyStats'; + latencyStats.classList.add('settingsContainer'); + const statisticsHeader = document.createElement('div'); statisticsHeader.id = 'statisticsHeader'; statisticsHeader.classList.add('settings-text'); statisticsHeader.classList.add('settingsHeader'); + const latencyStatsHeader = document.createElement('div'); + latencyStatsHeader.id = 'latencyStatsHeader'; + latencyStatsHeader.classList.add('settings-text'); + latencyStatsHeader.classList.add('settingsHeader'); + this._statsContentElement.appendChild(streamToolStats); streamToolStats.appendChild(controlStats); controlStats.appendChild(statistics); + controlStats.appendChild(latencyStats); + statistics.appendChild(statisticsHeader); + latencyStats.appendChild(latencyStatsHeader); + if (isSectionEnabled(this._config, StatsSections.SessionStats)) { - const sessionStats = document.createElement('div'); - sessionStats.innerHTML = StatsSections.SessionStats; - statisticsHeader.appendChild(sessionStats); + const sessionStatsText = document.createElement('div'); + sessionStatsText.innerHTML = StatsSections.SessionStats; + statisticsHeader.appendChild(sessionStatsText); } statistics.appendChild(this.statisticsContainer); + if (isSectionEnabled(this._config, StatsSections.LatencyStats)) { + const latencyStatsText = document.createElement('div'); + latencyStatsText.innerHTML = StatsSections.LatencyStats; + latencyStatsHeader.appendChild(latencyStatsText); + } + latencyStats.appendChild(this.latencyStatsContainer); + if (isSectionEnabled(this._config, StatsSections.LatencyTest)) { controlStats.appendChild(this.latencyTest.rootElement); } @@ -125,6 +152,16 @@ export class StatsPanel { return this._statisticsContainer; } + public get latencyStatsContainer(): HTMLElement { + if (!this._latencyStatsContainer) { + this._latencyStatsContainer = document.createElement('div'); + this._latencyStatsContainer.id = 'latencyStatsContainer'; + this._latencyStatsContainer.classList.add('d-none'); + this._latencyStatsContainer.appendChild(this.latencyResult); + } + return this._latencyStatsContainer; + } + public get statsResult(): HTMLElement { if (!this._statsResult) { this._statsResult = document.createElement('div'); @@ -134,6 +171,15 @@ export class StatsPanel { return this._statsResult; } + public get latencyResult(): HTMLElement { + if (!this._latencyResult) { + this._latencyResult = document.createElement('div'); + this._latencyResult.id = 'latencyResult'; + this._latencyResult.classList.add('StatsResult'); + } + return this._latencyResult; + } + public get statsCloseButton(): HTMLElement { if (!this._statsCloseButton) { this._statsCloseButton = document.createElement('div'); @@ -208,7 +254,7 @@ export class StatsPanel { } public handlePlayerCount(playerCount: number) { - this.addOrUpdateStat('PlayerCountStat', 'Players', playerCount.toString()); + this.addOrUpdateSessionStat('PlayerCountStat', 'Players', playerCount.toString()); } /** @@ -223,17 +269,17 @@ export class StatsPanel { // Inbound data const inboundData = MathUtils.formatBytes(stats.inboundVideoStats.bytesReceived, 2); - this.addOrUpdateStat('InboundDataStat', 'Received', inboundData); + this.addOrUpdateSessionStat('InboundDataStat', 'Received', inboundData); // Packets lost const packetsLostStat = Object.prototype.hasOwnProperty.call(stats.inboundVideoStats, 'packetsLost') ? numberFormat.format(stats.inboundVideoStats.packetsLost) : 'Chrome only'; - this.addOrUpdateStat('PacketsLostStat', 'Packets Lost', packetsLostStat); + this.addOrUpdateSessionStat('PacketsLostStat', 'Packets Lost', packetsLostStat); // Bitrate if (stats.inboundVideoStats.bitrate) { - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'VideoBitrateStat', 'Video Bitrate (kbps)', stats.inboundVideoStats.bitrate.toString() @@ -241,7 +287,7 @@ export class StatsPanel { } if (stats.inboundAudioStats.bitrate) { - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'AudioBitrateStat', 'Audio Bitrate (kbps)', stats.inboundAudioStats.bitrate.toString() @@ -250,23 +296,23 @@ export class StatsPanel { // Video resolution const resStat = - Object.prototype.hasOwnProperty.call(stats.inboundVideoStats, 'frameWidth') && - stats.inboundVideoStats.frameWidth && - Object.prototype.hasOwnProperty.call(stats.inboundVideoStats, 'frameHeight') && - stats.inboundVideoStats.frameHeight + stats.inboundVideoStats.frameWidth !== undefined && + stats.inboundVideoStats.frameWidth > 0 && + stats.inboundVideoStats.frameHeight !== undefined && + stats.inboundVideoStats.frameHeight > 0 ? stats.inboundVideoStats.frameWidth + 'x' + stats.inboundVideoStats.frameHeight : 'Chrome only'; - this.addOrUpdateStat('VideoResStat', 'Video resolution', resStat); + this.addOrUpdateSessionStat('VideoResStat', 'Video resolution', resStat); // Frames decoded - const framesDecoded = Object.prototype.hasOwnProperty.call(stats.inboundVideoStats, 'framesDecoded') - ? numberFormat.format(stats.inboundVideoStats.framesDecoded) - : 'Chrome only'; - this.addOrUpdateStat('FramesDecodedStat', 'Frames Decoded', framesDecoded); + if (stats.inboundVideoStats.framesDecoded !== undefined) { + const framesDecoded = numberFormat.format(stats.inboundVideoStats.framesDecoded); + this.addOrUpdateSessionStat('FramesDecodedStat', 'Frames Decoded', framesDecoded); + } // Framerate if (stats.inboundVideoStats.framesPerSecond) { - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'FramerateStat', 'Framerate', stats.inboundVideoStats.framesPerSecond.toString() @@ -274,14 +320,16 @@ export class StatsPanel { } // Frames dropped - this.addOrUpdateStat( - 'FramesDroppedStat', - 'Frames dropped', - stats.inboundVideoStats.framesDropped?.toString() - ); + if (stats.inboundVideoStats.framesDropped !== undefined) { + this.addOrUpdateSessionStat( + 'FramesDroppedStat', + 'Frames dropped', + stats.inboundVideoStats.framesDropped.toString() + ); + } if (stats.inboundVideoStats.codecId) { - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'VideoCodecStat', 'Video codec', // Split the codec to remove the Fmtp line @@ -290,7 +338,7 @@ export class StatsPanel { } if (stats.inboundAudioStats.codecId) { - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'AudioCodecStat', 'Audio codec', // Split the codec to remove the Fmtp line @@ -308,12 +356,12 @@ export class StatsPanel { stats.isNumber(activeCandidatePair.currentRoundTripTime) ? Math.ceil(activeCandidatePair.currentRoundTripTime * 1000).toString() : "Can't calculate"; - this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT); + this.addOrUpdateSessionStat('RTTStat', 'Net RTT (ms)', netRTT); } - this.addOrUpdateStat('DurationStat', 'Duration', stats.sessionStats.runTime); + this.addOrUpdateSessionStat('DurationStat', 'Duration', stats.sessionStats.runTime); - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'ControlsInputStat', 'Controls stream input', stats.sessionStats.controlsStreamInput @@ -324,7 +372,7 @@ export class StatsPanel { stats.sessionStats.videoEncoderAvgQP !== undefined && !Number.isNaN(stats.sessionStats.videoEncoderAvgQP) ) { - this.addOrUpdateStat( + this.addOrUpdateSessionStat( 'QPStat', 'Video quantization parameter', stats.sessionStats.videoEncoderAvgQP.toString() @@ -338,7 +386,7 @@ export class StatsPanel { if (latencyInfo.frameTiming !== undefined) { // Encoder latency if (latencyInfo.frameTiming.encoderLatencyMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'EncodeLatency', 'Encode latency (ms)', Math.ceil(latencyInfo.frameTiming.encoderLatencyMs).toString() @@ -347,7 +395,7 @@ export class StatsPanel { // Packetizer latency if (latencyInfo.frameTiming.packetizeLatencyMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'PacketizerLatency', 'Packetizer latency (ms)', Math.ceil(latencyInfo.frameTiming.packetizeLatencyMs).toString() @@ -356,7 +404,7 @@ export class StatsPanel { // Pacer latency if (latencyInfo.frameTiming.pacerLatencyMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'PacerLatency', 'Pacer latency (ms)', Math.ceil(latencyInfo.frameTiming.pacerLatencyMs).toString() @@ -365,7 +413,7 @@ export class StatsPanel { // Sender latency calculated using timing stats if (latencyInfo.frameTiming.captureToSendLatencyMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'CaptureToSend', 'Capture to send latency (ms)', Math.ceil(latencyInfo.frameTiming.captureToSendLatencyMs).toString() @@ -374,7 +422,7 @@ export class StatsPanel { } if (latencyInfo.senderLatencyMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'SenderSideLatency', 'Sender latency (ms)', Math.ceil(latencyInfo.senderLatencyMs).toString() @@ -382,7 +430,7 @@ export class StatsPanel { } if (latencyInfo.averageAssemblyDelayMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'AvgAssemblyDelay', 'Assembly delay (ms)', Math.ceil(latencyInfo.averageAssemblyDelayMs).toString() @@ -390,7 +438,7 @@ export class StatsPanel { } if (latencyInfo.averageDecodeLatencyMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'AvgDecodeDelay', 'Decode time (ms)', Math.ceil(latencyInfo.averageDecodeLatencyMs).toString() @@ -398,7 +446,7 @@ export class StatsPanel { } if (latencyInfo.averageJitterBufferDelayMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'AvgJitterBufferDelay', 'Jitter buffer (ms)', Math.ceil(latencyInfo.averageJitterBufferDelayMs).toString() @@ -406,7 +454,7 @@ export class StatsPanel { } if (latencyInfo.averageProcessingDelayMs !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'AvgProcessingDelay', 'Processing delay (ms)', Math.ceil(latencyInfo.averageProcessingDelayMs).toString() @@ -414,7 +462,7 @@ export class StatsPanel { } if (latencyInfo.averageE2ELatency !== undefined) { - this.addOrUpdateStat( + this.addOrUpdateLatencyStat( 'AvgE2ELatency', 'Total latency (ms)', Math.ceil(latencyInfo.averageE2ELatency).toString() @@ -427,11 +475,47 @@ export class StatsPanel { * @param id - The id of the stat to add/update. * @param stat - The contents of the stat. */ - public addOrUpdateStat(id: string, statLabel: string, stat: string) { - if (!isSectionEnabled(this._config, StatsSections.SessionStats)) { + public addOrUpdateSessionStat(id: string, statLabel: string, stat: string) { + this.addOrUpdateStat(StatsSections.SessionStats, id, statLabel, stat); + } + + /** + * Adds a new stat to the latency results in the DOM or updates an exiting stat. + * @param id - The id of the stat to add/update. + * @param stat - The contents of the stat. + */ + public addOrUpdateLatencyStat(id: string, statLabel: string, stat: string) { + this.addOrUpdateStat(StatsSections.LatencyStats, id, statLabel, stat); + } + + /** + * Adds a new stat to the stats results in the DOM or updates an exiting stat. + * @param sectionId - The section to add this stat too. + * @param id - The id of the stat to add/update. + * @param stat - The contents of the stat. + */ + private addOrUpdateStat(sectionId: StatsSectionsIds, id: string, statLabel: string, stat: string) { + if ( + sectionId === StatsSections.SessionStats && + !isSectionEnabled(this._config, StatsSections.SessionStats) + ) { + return; + } + + if ( + sectionId === StatsSections.LatencyStats && + !isSectionEnabled(this._config, StatsSections.LatencyStats) + ) { + return; + } + + // Only support session or latency stats being updated in this function currently + if (sectionId !== StatsSections.SessionStats && sectionId !== StatsSections.LatencyStats) { return; } + const parentElem: HTMLElement = + sectionId === StatsSections.SessionStats ? this.statsResult : this.latencyResult; const statHTML = `${statLabel}: ${stat}`; if (!this.statsMap.has(id)) { @@ -443,7 +527,7 @@ export class StatsPanel { newStat.element = document.createElement('div'); newStat.element.innerHTML = statHTML; // add the stat to the dom - this.statsResult.appendChild(newStat.element); + parentElem.appendChild(newStat.element); this.statsMap.set(id, newStat); } // update the existing stat diff --git a/Frontend/ui-library/src/UI/UIConfigurationTypes.ts b/Frontend/ui-library/src/UI/UIConfigurationTypes.ts index 89b39b334..69f051a4b 100644 --- a/Frontend/ui-library/src/UI/UIConfigurationTypes.ts +++ b/Frontend/ui-library/src/UI/UIConfigurationTypes.ts @@ -30,6 +30,7 @@ export type SettingsSectionsIds = (typeof SettingsSections)[SettingsSectionsKeys export class StatsSections { static SessionStats = 'Session Stats' as const; + static LatencyStats = 'Latency Stats' as const; static LatencyTest = 'Latency Test' as const; static DataChannelLatencyTest = 'Data Channel Latency Test' as const; } From e00a2e50ae11aa8b9838ce660245927107e73250 Mon Sep 17 00:00:00 2001 From: Luke Bermingham <1215582+lukehb@users.noreply.github.com> Date: Fri, 7 Feb 2025 15:58:16 +1000 Subject: [PATCH 25/27] Apply suggestions from code review --- .github/workflows/healthcheck-streaming.yml | 2 -- Extras/FrontendTests/tests/helpers.ts | 5 ----- Frontend/implementations/typescript/tsconfig.esm.json | 4 ++-- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/healthcheck-streaming.yml b/.github/workflows/healthcheck-streaming.yml index 05f89b595..d51cbeed4 100644 --- a/.github/workflows/healthcheck-streaming.yml +++ b/.github/workflows/healthcheck-streaming.yml @@ -121,8 +121,6 @@ jobs: - name: Wait for signalling to come up run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/api/status - - name: Wait for front end to come up - run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/ - name: Wait for streamer to come up run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/api/streamers/DefaultStreamer diff --git a/Extras/FrontendTests/tests/helpers.ts b/Extras/FrontendTests/tests/helpers.ts index 008cb046b..78190f4a7 100644 --- a/Extras/FrontendTests/tests/helpers.ts +++ b/Extras/FrontendTests/tests/helpers.ts @@ -31,11 +31,6 @@ export function delay(time: number) { }); } -export async function startStreaming(page: Page) { - await page.evaluate(()=> { - window.pixelStreaming.connect(); - }); -} export async function startAndWaitForVideo(page: Page) { await page.evaluate(()=> { diff --git a/Frontend/implementations/typescript/tsconfig.esm.json b/Frontend/implementations/typescript/tsconfig.esm.json index 595a922ce..2671e8eba 100644 --- a/Frontend/implementations/typescript/tsconfig.esm.json +++ b/Frontend/implementations/typescript/tsconfig.esm.json @@ -2,7 +2,7 @@ "extends": "./tsconfig.base.json", "compilerOptions": { "outDir": "./dist/esm", - "module": "ES6", - "moduleResolution": "nodenext" + "module": "es6", + "moduleResolution": "bundler" } } From 070bbf8ae52f66fbfae91efb7d06400752571c64 Mon Sep 17 00:00:00 2001 From: Luke Bermingham <1215582+lukehb@users.noreply.github.com> Date: Wed, 12 Feb 2025 16:58:56 +1300 Subject: [PATCH 26/27] Fix failing CI for build:esm job --- Frontend/implementations/typescript/tsconfig.esm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Frontend/implementations/typescript/tsconfig.esm.json b/Frontend/implementations/typescript/tsconfig.esm.json index 4edbe6cfc..2671e8eba 100644 --- a/Frontend/implementations/typescript/tsconfig.esm.json +++ b/Frontend/implementations/typescript/tsconfig.esm.json @@ -3,6 +3,6 @@ "compilerOptions": { "outDir": "./dist/esm", "module": "es6", - "moduleResolution": "nodenext" + "moduleResolution": "bundler" } } From ee5d7bd61f6a7ac48d2bdc828311aa9275d08922 Mon Sep 17 00:00:00 2001 From: Luke Bermingham <1215582+lukehb@users.noreply.github.com> Date: Wed, 12 Feb 2025 17:56:04 +1300 Subject: [PATCH 27/27] Made it clearer whether video timing or abs-capture-time is used --- .../tests/peerconnection.spec.ts | 49 ++++++++++++++++--- .../LatencyCalculator.ts | 28 +++++++---- .../PeerConnectionController.ts | 2 +- Frontend/ui-library/src/UI/StatsPanel.ts | 8 +-- 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 41459fe72..bb62d0bbc 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -96,7 +96,7 @@ test('Test abs-capture-time header extension found in PSInfra frontend', { expect(answer.sdp).toContain("abs-capture-time"); }); -test('Test latency calculation', { +test('Test video-timing header extension found in PSInfra frontend', { tag: ['@capture-time'], }, async ({ page, streamerPage, streamerId, browserName }) => { @@ -109,18 +109,52 @@ test('Test latency calculation', { await page.waitForLoadState("load"); - // Enable the flag for the capture extension - await page.evaluate(() => { - window.pixelStreaming.config.setFlagEnabled("EnableCaptureTimeExt", true); + // Wait for the sdp answer + let getSdpAnswer = new Promise(async (resolve) => { + + // Expose the resolve function to the browser context + await page.exposeFunction('resolveFromSdpAnswerPromise', resolve); + + page.evaluate(() => { + window.pixelStreaming.addEventListener("webRtcSdpAnswer", (e: WebRtcSdpAnswerEvent) => { + resolveFromSdpAnswerPromise(e.data.sdp); + }); + }); + }); + await helpers.startAndWaitForVideo(page); + const answer: RTCSessionDescriptionInit = await getSdpAnswer; + + expect(answer).toBeDefined(); + expect(answer.sdp).toBeDefined(); + + // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer + expect(answer.sdp).toContain("video-timing"); +}); + +test('Test latency calculation with video timing', { + tag: ['@video-timing'], +}, async ({ page, streamerPage, streamerId, browserName }) => { + + if(browserName !== 'chromium') { + // Chrome based browsers are the only ones that support. + test.skip(); + } + + await page.goto(`/?StreamerId=${streamerId}`); + + await page.waitForLoadState("load"); + await helpers.startAndWaitForVideo(page); // Wait for the latency info event to be fired let latencyInfo: LatencyInfo = await page.evaluate(() => { return new Promise((resolve) => { window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => { - if(e.data.latencyInfo && e.data.latencyInfo.senderLatencyMs && e.data.latencyInfo.rttMs) { + if(e.data.latencyInfo && e.data.latencyInfo.frameTiming && + e.data.latencyInfo.frameTiming.captureToSendLatencyMs && + e.data.latencyInfo.rttMs) { resolve(e.data.latencyInfo); } }); @@ -128,7 +162,8 @@ test('Test latency calculation', { }); expect(latencyInfo).toBeDefined(); - expect(latencyInfo.senderLatencyMs).toBeDefined(); + expect(latencyInfo.frameTiming).toBeDefined(); + expect(latencyInfo.frameTiming?.captureToSendLatencyMs).toBeDefined(); expect(latencyInfo.averageJitterBufferDelayMs).toBeDefined(); expect(latencyInfo.averageProcessingDelayMs).toBeDefined(); expect(latencyInfo.rttMs).toBeDefined(); @@ -136,7 +171,7 @@ test('Test latency calculation', { expect(latencyInfo.averageDecodeLatencyMs).toBeDefined(); // Sender side latency should be less than 500ms in pure CPU test - expect(latencyInfo.senderLatencyMs).toBeLessThanOrEqual(500) + expect(latencyInfo.frameTiming?.captureToSendLatencyMs).toBeLessThanOrEqual(500) // Expect jitter buffer/processing delay to be no greater than 500ms on local link expect(latencyInfo.averageJitterBufferDelayMs).toBeLessThanOrEqual(500); diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts index c67291eb1..c06f4be8b 100644 --- a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -145,20 +145,23 @@ export class LatencyCalculator { ); } - // If we could not calculate latency because `senderLatencyMs` was missing because of no SenderReport yet - // We can try to substitute frame timing capture to send latency instead. + // Calculate E2E latency using video-timing capture to send time + one way network latency + receiver-side latency if ( - latencyInfo.senderLatencyMs === undefined && latencyInfo.frameTiming !== undefined && - latencyInfo.frameTiming.captureToSendLatencyMs !== undefined + latencyInfo.frameTiming.captureToSendLatencyMs !== undefined && + latencyInfo.averageProcessingDelayMs !== undefined && + latencyInfo.rttMs !== undefined ) { - latencyInfo.senderLatencyMs = latencyInfo.frameTiming.captureToSendLatencyMs; + latencyInfo.averageE2ELatency = + latencyInfo.frameTiming.captureToSendLatencyMs + + latencyInfo.rttMs * 0.5 + + latencyInfo.averageProcessingDelayMs; } - // Calculate E2E latency as sender-side latency + one way network latency + receiver-side latency + // Calculate E2E latency as abs-capture-time capture to send latency + one way network latency + receiver-side latency if ( - latencyInfo.averageProcessingDelayMs !== undefined && latencyInfo.senderLatencyMs != undefined && + latencyInfo.averageProcessingDelayMs !== undefined && latencyInfo.rttMs !== undefined ) { latencyInfo.averageE2ELatency = @@ -353,12 +356,19 @@ export class LatencyCalculator { */ export class LatencyInfo { /** - * The time taken from sender frame capture to receiver frame receipt. + * The time taken from the moment a frame is done capturing to the moment it is sent over the network. * Note: This can only be calculated if both offer and answer contain the - * the RTP header extension for `abs-capture-time`. + * the RTP header extension for `video-timing` (Chrome only for now) */ public senderLatencyMs: number | undefined = undefined; + /** + * The time taken from the moment a frame is done capturing to the moment it is sent over the network. + * Note: This can only be calculated if both offer and answer contain the + * the RTP header extension for `abs-capture-time` (Chrome only for now) + */ + public senderLatencyAbsCaptureTimeMs: number | undefined = undefined; + /* The round trip time (milliseconds) between each sender->receiver->sender */ public rttMs: number | undefined = undefined; diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index cb2159b49..0942f86cf 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -272,7 +272,7 @@ export class PeerConnectionController { } isFirefox(): boolean { - return typeof (window as any).InstallTrigger !== 'undefined'; + return navigator.userAgent.indexOf('Firefox') > 0; } /** diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index 8521caa63..e1f34c43e 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -414,8 +414,8 @@ export class StatsPanel { // Sender latency calculated using timing stats if (latencyInfo.frameTiming.captureToSendLatencyMs !== undefined) { this.addOrUpdateLatencyStat( - 'CaptureToSend', - 'Capture to send latency (ms)', + 'VideoTimingCaptureToSend', + 'Post-capture to send latency (ms)', Math.ceil(latencyInfo.frameTiming.captureToSendLatencyMs).toString() ); } @@ -423,8 +423,8 @@ export class StatsPanel { if (latencyInfo.senderLatencyMs !== undefined) { this.addOrUpdateLatencyStat( - 'SenderSideLatency', - 'Sender latency (ms)', + 'AbsCaptureTimeToSendLatency', + 'Post-capture (abs-ct) to send latency (ms)', Math.ceil(latencyInfo.senderLatencyMs).toString() ); }