import Direction from '../clientmetricreport/ClientMetricReportDirection' ; import MediaType from '../clientmetricreport/ClientMetricReportMediaType' ; import ClientMetricReport from '../clientmetricreport/DefaultClientMetricReport' ; import ContentShareConstants from '../contentsharecontroller/ContentShareConstants' ; import Logger from '../logger/Logger' ; import { LogLevel } from '../logger/LogLevel' ; import DefaultVideoStreamIdSet from '../videostreamidset/DefaultVideoStreamIdSet' ; import VideoStreamIdSet from '../videostreamidset/VideoStreamIdSet' ; import VideoStreamDescription from '../videostreamindex/VideoStreamDescription' ; import VideoStreamIndex from '../videostreamindex/VideoStreamIndex' ; import DefaultVideoTile from '../videotile/DefaultVideoTile' ; import VideoTile from '../videotile/VideoTile' ; import VideoTileController from '../videotilecontroller/VideoTileController' ; import TargetDisplaySize from './TargetDisplaySize' ; import VideoDownlinkBandwidthPolicy from './VideoDownlinkBandwidthPolicy' ; import VideoDownlinkObserver from './VideoDownlinkObserver' ; import VideoPreference from './VideoPreference' ; import { VideoPreferences } from './VideoPreferences' ; import VideoPriorityBasedPolicyConfig from './VideoPriorityBasedPolicyConfig' ; internal class LinkMediaStats { constructor ( ) { } bandwidthEstimateKbps : number = 0 ; usedBandwidthKbps : number = 0 ; packetsLost : number = 0 ; nackCount : number = 0 ; rttMs : number = 0 ; } internal interface PolicyRates { targetDownlinkBitrate : number ; chosenTotalBitrate : number ; deltaToNextUpgrade : number ; } internal const enum RateProbeState { NotProbing = 'Not Probing' , ProbePending = 'Probe Pending' , Probing = 'Probing' } internal const enum UseReceiveSet { NewOptimal , PreviousOptimal , PreProbe } internal const enum NetworkEvent { Decrease , Increase } export default class VideoPriorityBasedPolicy implements VideoDownlinkBandwidthPolicy { private static readonly DEFAULT_BANDWIDTH_KBPS = 2800 ; private static readonly STARTUP_PERIOD_MS = 6000 ; private static readonly LARGE_RATE_CHANGE_TRIGGER_PERCENT = 20 ; private static readonly TARGET_RATE_CHANGE_TRIGGER_PERCENT = 15 ; private static readonly LOW_BITRATE_THRESHOLD_KBPS = 300 ; private static readonly MIN_TIME_BETWEEN_PROBE_MS = 5000 ; private static readonly MIN_TIME_BETWEEN_SUBSCRIBE_MS = 2000 ; private static readonly MAX_HOLD_BEFORE_PROBE_MS = 60000 ; private static readonly MAX_ALLOWED_PROBE_TIME_MS = 60000 ; protected tileController : VideoTileController | undefined ; protected videoPreferences : VideoPreferences | undefined ; protected defaultVideoPreferences : VideoPreferences | undefined ; protected shouldPauseTiles : boolean = true ; protected videoIndex : VideoStreamIndex ; protected videoPreferencesUpdated : boolean ; protected observerQueue : Set < VideoDownlinkObserver > = new Set < VideoDownlinkObserver > ( ) ; private logCount : number ; private optimalNonPausedReceiveStreams : VideoStreamDescription [ ] ; private optimalReceiveStreams : VideoStreamDescription [ ] ; private optimalReceiveSet : VideoStreamIdSet ; private subscribedReceiveSet : VideoStreamIdSet ; private subscribedReceiveStreams : VideoStreamDescription [ ] ; private preProbeReceiveStreams : VideoStreamDescription [ ] ; private preProbeNonPausedReceiveStreams : VideoStreamDescription [ ] ; private pausedStreamIds : VideoStreamIdSet ; private pausedBwAttendeeIds : Set < string > = new Set < string > ( ) ; private downlinkStats : LinkMediaStats ; private prevDownlinkStats : LinkMediaStats ; private prevRemoteInfos : VideoStreamDescription [ ] ; private rateProbeState : RateProbeState ; private startupPeriod : boolean ; private usingPrevTargetRate : boolean ; private prevTargetRateKbps : number ; private lastUpgradeRateKbps : number ; private firstEstimateTimestamp : number ; private lastSubscribeTimestamp : number ; private timeBeforeAllowSubscribeMs : number ; private probePendingStartTimestamp : number ; private timeBeforeAllowProbeMs : number ; private lastProbeTimestamp : number ; private probeFailed : boolean ; constructor ( protected logger , private videoPriorityBasedPolicyConfig = VideoPriorityBasedPolicyConfig . Default ) { this . reset ( ) ; } reset ( ) { this . optimalReceiveSet = new DefaultVideoStreamIdSet ( ) ; this . optimalReceiveStreams = [ ] ; this . optimalNonPausedReceiveStreams = [ ] ; this . subscribedReceiveSet = new DefaultVideoStreamIdSet ( ) ; this . subscribedReceiveStreams = [ ] ; this . videoPreferences = undefined ; this . defaultVideoPreferences = undefined ; this . shouldPauseTiles = true ; this . pausedStreamIds = new DefaultVideoStreamIdSet ( ) ; this . pausedBwAttendeeIds = new Set < string > ( ) ; this . videoPreferencesUpdated = false ; this . logCount = 0 ; this . startupPeriod = true ; this . usingPrevTargetRate = false ; this . rateProbeState = RateProbeState . NotProbing ; this . firstEstimateTimestamp = 0 ; this . lastUpgradeRateKbps = 0 ; this . timeBeforeAllowSubscribeMs = VideoPriorityBasedPolicy . MIN_TIME_BETWEEN_SUBSCRIBE_MS ; this . lastProbeTimestamp = Date . now ( ) ; this . timeBeforeAllowProbeMs = VideoPriorityBasedPolicy . MIN_TIME_BETWEEN_PROBE_MS ; this . downlinkStats = new LinkMediaStats ( ) ; this . prevDownlinkStats = new LinkMediaStats ( ) ; this . probeFailed = false ; } bindToTileController ( tileController ) { this . tileController = tileController ; this . logger . info ( 'tileController bound' ) ; } chooseRemoteVideoSources ( preferences ) { if ( this . videoPreferences ?. equals ( preferences ) ) { return ; } this . videoPreferences = preferences ; this . videoPreferencesUpdated = true ; this . logger . info ( `bwe: setVideoPreferences bwe: new preferences: ${ JSON . stringify ( preferences ) }` ) ; return ; } updateIndex ( videoIndex ) { this . videoIndex = videoIndex ; if ( ! this . videoPreferences ) { this . updateDefaultVideoPreferences ( ) ; } } private updateDefaultVideoPreferences ( ) { const attendeeIds = new Set < string > ( ) ; for ( const stream of this . videoIndex . remoteStreamDescriptions ( ) ) { attendeeIds . add ( stream . attendeeId ) ; } const prefs = VideoPreferences . prepare ( ) ; const numAttendees = attendeeIds . size ; let targetDisplaySize = TargetDisplaySize . High ; if ( numAttendees > 8 ) { targetDisplaySize = TargetDisplaySize . Low ; } else if ( numAttendees > 4 ) { targetDisplaySize = TargetDisplaySize . Medium ; } for ( const attendeeId of attendeeIds ) { prefs . add ( new VideoPreference ( attendeeId , 1 , targetDisplaySize ) ) ; } this . defaultVideoPreferences = prefs . build ( ) ; } updateMetrics ( clientMetricReport ) { if ( ! this . videoIndex || this . videoIndex . allStreams ( ) . empty ( ) ) { return ; } this . prevDownlinkStats = this . downlinkStats ; this . downlinkStats = new LinkMediaStats ( ) ; const metricReport = clientMetricReport . getObservableMetrics ( ) ; if ( ! isNaN ( metricReport . availableIncomingBitrate ) ) { this . downlinkStats . bandwidthEstimateKbps = metricReport . availableIncomingBitrate / 1000 ; } else { this . downlinkStats . bandwidthEstimateKbps = metricReport . availableReceiveBandwidth / 1000 ; } for ( const ssrcStr in clientMetricReport . streamMetricReports ) { const ssrc = Number . parseInt ( ssrcStr , 10 ) ; const metrics = clientMetricReport . streamMetricReports [ ssrc ] ; if ( metrics . direction === Direction . DOWNSTREAM && metrics . mediaType === MediaType . VIDEO ) { if ( metrics . currentMetrics . hasOwnProperty ( 'googNacksSent' ) ) { this . downlinkStats . nackCount += clientMetricReport . countPerSecond ( 'googNacksSent' , ssrc ) ; } else if ( metrics . currentMetrics . hasOwnProperty ( 'nackCount' ) ) { this . downlinkStats . nackCount += clientMetricReport . countPerSecond ( 'nackCount' , ssrc ) ; } if ( metrics . currentMetrics . hasOwnProperty ( 'packetsLost' ) ) { this . downlinkStats . packetsLost += clientMetricReport . countPerSecond ( 'packetsLost' , ssrc ) ; } if ( metrics . currentMetrics . hasOwnProperty ( 'bytesReceived' ) ) { this . downlinkStats . usedBandwidthKbps += clientMetricReport . bitsPerSecond ( 'bytesReceived' , ssrc ) / 1000 ; } } } } wantsResubscribe ( ) { this . calculateOptimalReceiveSet ( ) ; return ! this . subscribedReceiveSet . equal ( this . optimalReceiveSet ) ; } chooseSubscriptions ( ) { if ( ! this . subscribedReceiveSet . equal ( this . optimalReceiveSet ) ) { this . lastSubscribeTimestamp = Date . now ( ) ; } this . subscribedReceiveSet = this . optimalReceiveSet . clone ( ) ; this . subscribedReceiveStreams = this . optimalReceiveStreams . slice ( ) ; return this . subscribedReceiveSet . clone ( ) ; } addObserver ( observer ) { this . observerQueue . add ( observer ) ; } removeObserver ( observer ) { this . observerQueue . delete ( observer ) ; } forEachObserver ( observerFunc ) { for ( const observer of this . observerQueue ) { observerFunc ( observer ) ; } } setVideoPriorityBasedPolicyConfigs ( config ) { this . videoPriorityBasedPolicyConfig = config ; } private static readonly MINIMUM_DELAY = VideoPriorityBasedPolicy . MIN_TIME_BETWEEN_SUBSCRIBE_MS ; private static readonly MAXIMUM_DELAY = 8000 ; private getSubscribeDelay ( event , numberOfParticipants ) { let subscribeDelay = VideoPriorityBasedPolicy . MINIMUM_DELAY ; const range = VideoPriorityBasedPolicy . MAXIMUM_DELAY - VideoPriorityBasedPolicy . MINIMUM_DELAY ; const responseFactor = this . videoPriorityBasedPolicyConfig . networkIssueResponseDelayFactor ; const recoveryFactor = this . videoPriorityBasedPolicyConfig . networkIssueRecoveryDelayFactor ; switch ( event ) { case NetworkEvent . Decrease : subscribeDelay += range * responseFactor * ( 1 + numberOfParticipants / 10 ) ; subscribeDelay = Math . min ( VideoPriorityBasedPolicy . MAXIMUM_DELAY , subscribeDelay ) ; break ; case NetworkEvent . Increase : subscribeDelay += range * recoveryFactor ; break ; } return subscribeDelay ; } protected calculateOptimalReceiveStreams ( ) { const chosenStreams = [ ] ; const remoteInfos = this . videoIndex . remoteStreamDescriptions ( ) ; if ( remoteInfos . length === 0 || this . videoPreferences ?. isEmpty ( ) ) { this . optimalReceiveStreams = [ ] ; return ; } const lastProbeState = this . rateProbeState ; this . cleanBwPausedTiles ( remoteInfos ) ; this . handleAppPausedStreams ( chosenStreams , remoteInfos ) ; const sameStreamChoices = this . availStreamsSameAsLast ( remoteInfos ) ; const noMajorChange = ! this . startupPeriod && sameStreamChoices ; if ( noMajorChange && this . probeFailed && Date . now ( ) - this . lastSubscribeTimestamp < this . timeBeforeAllowSubscribeMs ) { return ; } this . probeFailed = false ; remoteInfos . sort ( ( a , b ) => { if ( a . maxBitrateKbps === b . maxBitrateKbps ) { return a . streamId - b . streamId ; } return a . maxBitrateKbps - b . maxBitrateKbps ; } ) ; for ( const info of remoteInfos ) { if ( info . avgBitrateKbps === 0 || info . avgBitrateKbps > info . maxBitrateKbps ) { if ( info . attendeeId . endsWith ( ContentShareConstants . Modality ) && info . maxBitrateKbps < 100 ) { info . maxBitrateKbps = info . avgBitrateKbps ; } else { info . avgBitrateKbps = info . maxBitrateKbps ; } } } const rates = { targetDownlinkBitrate : 0 , chosenTotalBitrate : 0 , deltaToNextUpgrade : 0 , } ; rates . targetDownlinkBitrate = this . determineTargetRate ( ) ; const numberOfParticipants = this . subscribedReceiveSet . size ( ) ; const prevEstimated = this . prevDownlinkStats . bandwidthEstimateKbps ; const currEstimated = this . downlinkStats . bandwidthEstimateKbps ; if ( currEstimated > prevEstimated ) { this . timeBeforeAllowSubscribeMs = this . getSubscribeDelay ( NetworkEvent . Increase , numberOfParticipants ) ; } else if ( currEstimated < prevEstimated ) { this . timeBeforeAllowSubscribeMs = this . getSubscribeDelay ( NetworkEvent . Decrease , numberOfParticipants ) ; } if ( noMajorChange && Date . now ( ) - this . lastSubscribeTimestamp < this . timeBeforeAllowSubscribeMs ) { return ; } const upgradeStream = this . priorityPolicy ( rates , remoteInfos , chosenStreams ) ; let subscriptionChoice = UseReceiveSet . NewOptimal ; if ( ! this . startupPeriod && sameStreamChoices ) { if ( this . rateProbeState === RateProbeState . Probing ) { subscriptionChoice = this . handleProbe ( chosenStreams , rates . targetDownlinkBitrate ) ; } else if ( rates . deltaToNextUpgrade !== 0 ) { subscriptionChoice = this . maybeOverrideOrProbe ( chosenStreams , rates , upgradeStream ) ; } } else { this . setProbeState ( RateProbeState . NotProbing ) ; this . lastUpgradeRateKbps = 0 ; } this . prevRemoteInfos = remoteInfos ; this . videoPreferencesUpdated = false ; if ( subscriptionChoice === UseReceiveSet . PreviousOptimal ) { this . logger . info ( `bwe: keepSameSubscriptions stats:${ JSON . stringify ( this . downlinkStats ) }` ) ; this . prevTargetRateKbps = rates . targetDownlinkBitrate ; return ; } if ( subscriptionChoice === UseReceiveSet . PreProbe ) { const subscribedRate = this . calculateSubscribeRate ( this . preProbeNonPausedReceiveStreams ) ; this . optimalReceiveStreams = this . preProbeReceiveStreams . slice ( ) ; this . processBwPausedStreams ( remoteInfos , this . preProbeNonPausedReceiveStreams ) ; this . logger . info ( 'bwe: Use Pre-Probe subscription subscribedRate:' + subscribedRate ) ; return ; } this . optimalNonPausedReceiveStreams = chosenStreams . slice ( ) ; const lastNumberPaused = this . pausedBwAttendeeIds . size ; this . processBwPausedStreams ( remoteInfos , chosenStreams ) ; if ( this . logger . getLogLevel ( ) <= LogLevel . INFO && ( this . logCount % 15 === 0 || this . rateProbeState !== lastProbeState || this . optimalReceiveStreams . length !== chosenStreams . length || lastNumberPaused !== this . pausedBwAttendeeIds . size ) ) { this . logger . info ( this . policyStateLogStr ( remoteInfos , rates . targetDownlinkBitrate ) ) ; this . logCount = 0 ; } this . logCount ++ ; this . prevTargetRateKbps = rates . targetDownlinkBitrate ; this . optimalReceiveStreams = chosenStreams . slice ( ) ; } protected calculateOptimalReceiveSet ( ) { const streamSelectionSet = new DefaultVideoStreamIdSet ( ) ; this . calculateOptimalReceiveStreams ( ) ; for ( const stream of this . optimalReceiveStreams ) { streamSelectionSet . add ( stream . streamId ) ; } if ( ! this . optimalReceiveSet . equal ( streamSelectionSet ) ) { const subscribedRate = this . calculateSubscribeRate ( this . optimalReceiveStreams ) ; this . logger . info ( `bwe: new streamSelection: ${ JSON . stringify ( streamSelectionSet ) } subscribedRate:${ subscribedRate }` ) ; } this . optimalReceiveSet = streamSelectionSet ; } private determineTargetRate ( ) { let targetBitrate = 0 ; const now = Date . now ( ) ; if ( this . downlinkStats . bandwidthEstimateKbps !== 0 ) { if ( this . firstEstimateTimestamp === 0 ) { this . firstEstimateTimestamp = now ; } if ( this . startupPeriod ) { if ( this . downlinkStats . bandwidthEstimateKbps > VideoPriorityBasedPolicy . DEFAULT_BANDWIDTH_KBPS || this . downlinkStats . packetsLost > 0 || ( now - this . firstEstimateTimestamp > VideoPriorityBasedPolicy . STARTUP_PERIOD_MS && this . downlinkStats . bandwidthEstimateKbps <= this . prevDownlinkStats . bandwidthEstimateKbps ) ) { this . startupPeriod = false ; this . prevTargetRateKbps = this . downlinkStats . bandwidthEstimateKbps ; } } if ( this . startupPeriod ) { targetBitrate = VideoPriorityBasedPolicy . DEFAULT_BANDWIDTH_KBPS ; } else { targetBitrate = this . downlinkStats . bandwidthEstimateKbps ; } } else { if ( this . firstEstimateTimestamp === 0 ) { targetBitrate = VideoPriorityBasedPolicy . DEFAULT_BANDWIDTH_KBPS ; } else { targetBitrate = this . prevTargetRateKbps ; } } if ( ! this . startupPeriod && ( ( this . usingPrevTargetRate && this . downlinkStats . bandwidthEstimateKbps < this . prevTargetRateKbps ) || this . downlinkStats . bandwidthEstimateKbps < ( this . prevTargetRateKbps * ( 100 - VideoPriorityBasedPolicy . LARGE_RATE_CHANGE_TRIGGER_PERCENT ) ) / 100 || this . downlinkStats . bandwidthEstimateKbps < ( this . downlinkStats . usedBandwidthKbps * VideoPriorityBasedPolicy . LARGE_RATE_CHANGE_TRIGGER_PERCENT ) / 100 ) && this . downlinkStats . packetsLost === 0 ) { this . logger . debug ( ( ) => { return 'bwe: ValidateRate: Using Previous rate ' + this . prevTargetRateKbps ; } ) ; this . usingPrevTargetRate = true ; targetBitrate = this . prevTargetRateKbps ; } else { this . usingPrevTargetRate = false ; } return targetBitrate ; } private setProbeState ( newState ) { if ( this . rateProbeState === newState ) { return false ; } const now = Date . now ( ) ; switch ( newState ) { case RateProbeState . NotProbing : this . probePendingStartTimestamp = 0 ; break ; case RateProbeState . ProbePending : if ( this . lastProbeTimestamp === 0 || now - this . lastProbeTimestamp > VideoPriorityBasedPolicy . MIN_TIME_BETWEEN_PROBE_MS ) { this . probePendingStartTimestamp = now ; } else { return false ; } break ; case RateProbeState . Probing : if ( now - this . probePendingStartTimestamp > this . timeBeforeAllowProbeMs ) { this . lastProbeTimestamp = now ; this . preProbeReceiveStreams = this . subscribedReceiveStreams . slice ( ) ; this . preProbeNonPausedReceiveStreams = this . optimalNonPausedReceiveStreams ; this . timeBeforeAllowProbeMs = Math . min ( this . timeBeforeAllowProbeMs * 2 , VideoPriorityBasedPolicy . MAX_HOLD_BEFORE_PROBE_MS ) ; } else { return false ; } break ; } this . logger . info ( 'bwe: setProbeState to ' + newState + ' from ' + this . rateProbeState ) ; this . rateProbeState = newState ; return true ; } private upgradeToStream ( chosenStreams , upgradeStream ) { for ( let i = 0 ; i < chosenStreams . length ; i ++ ) { if ( chosenStreams [ i ] . groupId === upgradeStream . groupId ) { const diffRate = upgradeStream . avgBitrateKbps - chosenStreams [ i ] . avgBitrateKbps ; this . logger . info ( 'bwe: upgradeStream from ' + JSON . stringify ( chosenStreams [ i ] ) + ' to ' + upgradeStream ) ; this . lastUpgradeRateKbps = diffRate ; chosenStreams [ i ] = upgradeStream ; return diffRate ; } } chosenStreams . push ( upgradeStream ) ; this . lastUpgradeRateKbps = upgradeStream . avgBitrateKbps ; return this . lastUpgradeRateKbps ; } private handleProbe ( chosenStreams , targetDownlinkBitrate ) { if ( Date . now ( ) - this . lastProbeTimestamp > VideoPriorityBasedPolicy . MAX_ALLOWED_PROBE_TIME_MS ) { this . logger . info ( `bwe: Canceling probe due to timeout` ) ; this . setProbeState ( RateProbeState . NotProbing ) ; return UseReceiveSet . NewOptimal ; } if ( this . downlinkStats . packetsLost > 0 ) { this . setProbeState ( RateProbeState . NotProbing ) ; this . logger . info ( `bwe: Canceling probe due to network loss` ) ; this . probeFailed = true ; this . timeBeforeAllowSubscribeMs = Math . max ( VideoPriorityBasedPolicy . MIN_TIME_BETWEEN_SUBSCRIBE_MS , this . timeBeforeAllowSubscribeMs ) * 3 ; return UseReceiveSet . PreProbe ; } const subscribedRate = this . calculateSubscribeRate ( this . optimalReceiveStreams ) ; if ( this . chosenStreamsSameAsLast ( chosenStreams ) || targetDownlinkBitrate > subscribedRate ) { let avgRate = 0 ; for ( const chosenStream of chosenStreams ) { avgRate += chosenStream . avgBitrateKbps ; } if ( targetDownlinkBitrate > avgRate ) { this . logger . info ( `bwe: Probe successful` ) ; this . setProbeState ( RateProbeState . NotProbing ) ; this . timeBeforeAllowProbeMs = VideoPriorityBasedPolicy . MIN_TIME_BETWEEN_PROBE_MS ; return UseReceiveSet . NewOptimal ; } } return UseReceiveSet . PreviousOptimal ; } private maybeOverrideOrProbe ( chosenStreams , rates , upgradeStream ) { const sameSubscriptions = this . chosenStreamsSameAsLast ( chosenStreams ) ; let useLastSubscriptions = UseReceiveSet . NewOptimal ; const triggerPercent = rates . targetDownlinkBitrate > VideoPriorityBasedPolicy . LOW_BITRATE_THRESHOLD_KBPS ? VideoPriorityBasedPolicy . TARGET_RATE_CHANGE_TRIGGER_PERCENT : VideoPriorityBasedPolicy . TARGET_RATE_CHANGE_TRIGGER_PERCENT * 2 ; const minTargetBitrateDelta = ( rates . targetDownlinkBitrate * triggerPercent ) / 100 ; if ( ! sameSubscriptions && Math . abs ( rates . targetDownlinkBitrate - this . prevTargetRateKbps ) < minTargetBitrateDelta ) { this . logger . info ( 'bwe: MaybeOverrideOrProbe: Reuse last decision based on delta rate. {' + JSON . stringify ( this . subscribedReceiveSet ) + `}` ) ; useLastSubscriptions = UseReceiveSet . PreviousOptimal ; } if ( this . downlinkStats . packetsLost > this . prevDownlinkStats . packetsLost ) { this . setProbeState ( RateProbeState . NotProbing ) ; this . lastUpgradeRateKbps = 0 ; return useLastSubscriptions ; } if ( sameSubscriptions || useLastSubscriptions === UseReceiveSet . PreviousOptimal ) { switch ( this . rateProbeState ) { case RateProbeState . NotProbing : this . setProbeState ( RateProbeState . ProbePending ) ; break ; case RateProbeState . ProbePending : if ( this . setProbeState ( RateProbeState . Probing ) ) { this . upgradeToStream ( chosenStreams , upgradeStream ) ; useLastSubscriptions = UseReceiveSet . NewOptimal ; } break ; } } else { this . setProbeState ( RateProbeState . NotProbing ) ; } return useLastSubscriptions ; } private calculateSubscribeRate ( streams ) { let subscribeRate = 0 ; for ( const stream of streams ) { if ( ! this . pausedStreamIds . contain ( stream . streamId ) && ! this . pausedBwAttendeeIds . has ( stream . attendeeId ) ) { subscribeRate += stream . maxBitrateKbps ; } } return subscribeRate ; } private handleAppPausedStreams ( chosenStreams , remoteInfos ) { if ( ! this . tileController ) { this . logger . warn ( 'tileController not found!' ) ; return ; } this . pausedStreamIds = new DefaultVideoStreamIdSet ( ) ; const remoteTiles = this . tileController . getAllRemoteVideoTiles ( ) ; for ( const tile of remoteTiles ) { const state = tile . state ( ) ; if ( state . paused && ! this . pausedBwAttendeeIds . has ( state . boundAttendeeId ) ) { let j = remoteInfos . length ; while ( j -- ) { if ( remoteInfos [ j ] . attendeeId === state . boundAttendeeId ) { this . logger . info ( 'bwe: removed paused attendee ' + state . boundAttendeeId + ' streamId: ' + remoteInfos [ j ] . streamId ) ; this . pausedStreamIds . add ( remoteInfos [ j ] . streamId ) ; if ( this . subscribedReceiveSet . contain ( remoteInfos [ j ] . streamId ) ) { chosenStreams . push ( remoteInfos [ j ] ) ; } remoteInfos . splice ( j , 1 ) ; } } } } } private processBwPausedStreams ( remoteInfos , chosenStreams ) { if ( ! this . tileController ) { this . logger . warn ( 'tileController not found!' ) ; return ; } const preferences = this . getCurrentVideoPreferences ( ) ; if ( preferences && this . shouldPauseTiles ) { const videoTiles = this . tileController . getAllVideoTiles ( ) ; for ( const preference of preferences ) { const videoTile = this . getVideoTileForAttendeeId ( preference . attendeeId , videoTiles ) as DefaultVideoTile ; const paused = videoTile ?. state ( ) . paused || false ; if ( ! chosenStreams . some ( stream => stream . attendeeId === preference . attendeeId ) ) { const streamExists = remoteInfos . some ( stream => stream . attendeeId === preference . attendeeId ) ; if ( videoTile && streamExists ) { const info = this . optimalReceiveStreams . find ( stream => stream . attendeeId === preference . attendeeId ) ; if ( info !== undefined ) { if ( ! paused ) { this . logger . info ( `bwe: pausing streamId ${ info . streamId } attendee ${ preference . attendeeId } due to bandwidth` ) ; this . forEachObserver ( observer => { observer . tileWillBePausedByDownlinkPolicy ( videoTile . id ( ) ) ; } ) ; this . tileController . pauseVideoTile ( videoTile . id ( ) ) ; } chosenStreams . push ( info ) ; } this . pausedBwAttendeeIds . add ( preference . attendeeId ) ; } else if ( streamExists ) { const newTile = this . tileController . addVideoTile ( ) ; newTile . bindVideoStream ( preference . attendeeId , false , null , 0 , 0 , 0 , null ) ; this . forEachObserver ( observer => { observer . tileWillBePausedByDownlinkPolicy ( newTile . id ( ) ) ; } ) ; newTile . pause ( ) ; this . logger . info ( `bwe: Created video tile ${ newTile . id ( ) } for bw paused attendee ${ preference . attendeeId }` ) ; this . pausedBwAttendeeIds . add ( preference . attendeeId ) ; } } else if ( paused && this . pausedBwAttendeeIds . has ( preference . attendeeId ) ) { this . logger . info ( `bwe: unpausing attendee ${ preference . attendeeId } due to bandwidth` ) ; this . forEachObserver ( observer => { observer . tileWillBeUnpausedByDownlinkPolicy ( videoTile . id ( ) ) ; } ) ; this . tileController . unpauseVideoTile ( videoTile . id ( ) ) ; this . pausedBwAttendeeIds . delete ( preference . attendeeId ) ; } } } } private cleanBwPausedTiles ( remoteInfos ) { if ( ! this . tileController ) { this . logger . warn ( 'tileController not found!' ) ; return ; } const tiles = this . tileController . getAllRemoteVideoTiles ( ) ; const preferences = this . getCurrentVideoPreferences ( ) ; for ( const tile of tiles ) { const state = tile . state ( ) ; if ( ! state . boundVideoStream ) { if ( ! remoteInfos . some ( stream => stream . attendeeId === state . boundAttendeeId ) ) { this . tileController . removeVideoTile ( state . tileId ) ; this . logger . info ( `bwe: Removed video tile ${ state . tileId } for bw paused attendee ${ state . boundAttendeeId }` ) ; } else if ( preferences !== undefined && ! preferences . some ( pref => pref . attendeeId === state . boundAttendeeId ) ) { this . tileController . removeVideoTile ( state . tileId ) ; } } } } private priorityPolicy ( rates , remoteInfos , chosenStreams ) { let upgradeStream ; const videoPreferences = this . getCurrentVideoPreferences ( ) ; const highestPriority = videoPreferences . highestPriority ( ) ; let nextPriority ; let priority = highestPriority ; while ( priority !== - 1 ) { nextPriority = - 1 ; for ( const preference of videoPreferences ) { if ( preference . priority === priority ) { for ( const info of remoteInfos ) { if ( info . attendeeId === preference . attendeeId ) { if ( ! chosenStreams . some ( stream => stream . groupId === info . groupId ) ) { if ( rates . chosenTotalBitrate + info . avgBitrateKbps <= rates . targetDownlinkBitrate ) { chosenStreams . push ( info ) ; rates . chosenTotalBitrate += info . avgBitrateKbps ; } else if ( rates . deltaToNextUpgrade === 0 ) { rates . deltaToNextUpgrade = info . avgBitrateKbps ; upgradeStream = info ; } } } } } else { if ( preference . priority > priority ) { nextPriority = preference . priority ; break ; } } } for ( const preference of videoPreferences ) { if ( preference . priority === priority ) { for ( const info of remoteInfos ) { if ( info . attendeeId === preference . attendeeId ) { const index = chosenStreams . findIndex ( stream => stream . groupId === info . groupId && stream . maxBitrateKbps < info . maxBitrateKbps ) ; if ( index !== - 1 ) { const increaseKbps = info . avgBitrateKbps - chosenStreams [ index ] . avgBitrateKbps ; if ( this . hasSimulcastStreams ( remoteInfos , info . attendeeId , info . groupId ) && this . canUpgrade ( info . avgBitrateKbps , preference . targetSizeToBitrateKbps ( preference . targetSize ) ) ) { this . logger . info ( `bwe: attendee: ${ info . attendeeId } group: ${ info . groupId } has simulcast and can upgrade avgBitrate: ${ info . avgBitrateKbps } target: ${ preference . targetSizeToBitrateKbps ( preference . targetSize ) } targetTotalBitrate: ${ rates . targetDownlinkBitrate }` ) ; if ( rates . chosenTotalBitrate + increaseKbps <= rates . targetDownlinkBitrate ) { rates . chosenTotalBitrate += increaseKbps ; chosenStreams [ index ] = info ; } else if ( rates . deltaToNextUpgrade === 0 ) { rates . deltaToNextUpgrade = increaseKbps ; upgradeStream = info ; } } else { this . logger . info ( 'bwe: cannot upgrade stream quality beyond target size' ) ; } } } } } else { if ( preference . priority > priority ) { break ; } } } if ( priority === highestPriority && rates . deltaToNextUpgrade !== 0 ) { break ; } priority = nextPriority ; } return upgradeStream ; } private getVideoTileForAttendeeId ( attendeeId , videoTiles ) { for ( const tile of videoTiles ) { const state = tile . state ( ) ; if ( state . boundAttendeeId === attendeeId ) { return tile ; } } return null ; } private canUpgrade ( bitrateKbp , targetBitrateKbp ) { if ( bitrateKbp <= targetBitrateKbp ) { this . logger . info ( `bwe: canUpgrade: bitrateKbp: ${ bitrateKbp } targetBitrateKbp: ${ targetBitrateKbp }` ) ; return true ; } this . logger . info ( `bwe: cannot Upgrade: bitrateKbp: ${ bitrateKbp } targetBitrateKbp: ${ targetBitrateKbp }` ) ; return false ; } private hasSimulcastStreams ( remoteInfos , attendeeId , groupId ) { let streamCount = 0 ; for ( const info of remoteInfos ) { if ( info . attendeeId === attendeeId && info . groupId === groupId ) { streamCount ++ ; } } this . logger . info ( `bwe: attendeeId: ${ attendeeId } groupId: ${ groupId } hasSimulcastStreams: streamCount: ${ streamCount }` ) ; return streamCount > 1 ; } private availStreamsSameAsLast ( remoteInfos ) { if ( this . prevRemoteInfos === undefined || remoteInfos . length !== this . prevRemoteInfos . length || this . videoPreferencesUpdated === true ) { return false ; } for ( const info of remoteInfos ) { const infoMatch = this . prevRemoteInfos . find ( prevInfo => prevInfo . groupId === info . groupId && prevInfo . streamId === info . streamId && prevInfo . maxBitrateKbps === info . maxBitrateKbps ) ; if ( infoMatch === undefined ) { return false ; } } return true ; } private chosenStreamsSameAsLast ( chosenStreams ) { if ( this . optimalNonPausedReceiveStreams . length !== chosenStreams . length ) { return false ; } for ( const lastStream of this . optimalNonPausedReceiveStreams ) { if ( ! chosenStreams . some ( stream => stream . streamId === lastStream . streamId ) ) { return false ; } } return true ; } private policyStateLogStr ( remoteInfos , targetDownlinkBitrate ) { const subscribedRate = this . calculateSubscribeRate ( this . optimalReceiveStreams ) ; const optimalReceiveSet = { targetBitrate : targetDownlinkBitrate , subscribedRate : subscribedRate , probeState : this . rateProbeState , startupPeriod : this . startupPeriod , } ; let remoteInfoStr = `remoteInfos: [` ; for ( const info of remoteInfos ) { remoteInfoStr += `{grpId:${ info . groupId } strId:${ info . streamId } maxBr:${ info . maxBitrateKbps } avgBr:${ info . avgBitrateKbps }}, ` ; } remoteInfoStr += `]` ; let logString = `bwe: optimalReceiveSet ${ JSON . stringify ( optimalReceiveSet ) }\n` + `bwe:   prev ${ JSON . stringify ( this . prevDownlinkStats ) }\n` + `bwe:   now  ${ JSON . stringify ( this . downlinkStats ) }\n` + `bwe:   ${ remoteInfoStr }\n` ; if ( this . pausedStreamIds . size ( ) > 0 || this . pausedBwAttendeeIds . size > 0 ) { logString += `bwe:   paused: app stream ids ${ JSON . stringify ( this . pausedStreamIds ) }  bw attendees { ${ Array . from ( this . pausedBwAttendeeIds ) . join ( ' ' ) } }\n` ; } if ( this . videoPreferences ) { logString += `bwe:   preferences: ${ JSON . stringify ( this . videoPreferences ) }` ; } else { logString += `bwe:   default preferences: ${ JSON . stringify ( this . defaultVideoPreferences ) }` ; } return logString ; } private getCurrentVideoPreferences ( ) { return this . videoPreferences || this . defaultVideoPreferences ; } }