import AudioVideoController from '../audiovideocontroller/AudioVideoController' ; import DefaultBrowserBehavior from '../browserbehavior/DefaultBrowserBehavior' ; import ExtendedBrowserBehavior from '../browserbehavior/ExtendedBrowserBehavior' ; import type { Destroyable } from '../destroyable/Destroyable' ; import DeviceChangeObserver from '../devicechangeobserver/DeviceChangeObserver' ; import Logger from '../logger/Logger' ; import DefaultMediaDeviceFactory from '../mediadevicefactory/DefaultMediaDeviceFactory' ; import DeviceControllerBasedMediaStreamBroker from '../mediastreambroker/DeviceControllerBasedMediaStreamBroker' ; import AsyncScheduler from '../scheduler/AsyncScheduler' ; import IntervalScheduler from '../scheduler/IntervalScheduler' ; import { Maybe } from '../utils/Types' ; import DefaultVideoTile from '../videotile/DefaultVideoTile' ; import AudioInputDevice from './AudioInputDevice' ; import AudioNodeSubgraph from './AudioNodeSubgraph' ; import AudioTransformDevice , { isAudioTransformDevice } from './AudioTransformDevice' ; import Device from './Device' ; import DeviceSelection from './DeviceSelection' ; import GetUserMediaError from './GetUserMediaError' ; import NotFoundError from './NotFoundError' ; import NotReadableError from './NotReadableError' ; import OverconstrainedError from './OverconstrainedError' ; import PermissionDeniedError from './PermissionDeniedError' ; import RemovableAnalyserNode from './RemovableAnalyserNode' ; import TypeError from './TypeError' ; import VideoInputDevice from './VideoInputDevice' ; import VideoQualitySettings from './VideoQualitySettings' ; import VideoTransformDevice , { isVideoTransformDevice } from './VideoTransformDevice' ; type Thunk = ( ) => void ; function fillSMPTEColorBars ( canvas , xShift ) { const w = canvas . width ; const h = canvas . height ; const h1 = ( h * 2 ) / 3 ; const h2 = ( h * 3 ) / 4 ; const h3 = h ; const top = [ '#c0c0c0' , '#c0c000' , '#00c0c0' , '#00c000' , '#c000c0' , '#c00000' , '#0000c0' ] ; const middle = [ '#0000c0' , '#000000' , '#c000c0' , '#000000' , '#00c0c0' , '#000000' , '#c0c0c0' ] ; const bottom = [ '#00214c' , '#ffffff' , '#32006a' , '#131313' , '#090909' , '#131313' , '#1d1d1d' , '#131313' , ] ; const bottomX = [ w * 0 , ( ( w * 1 ) / 4 ) * ( 5 / 7 ) , ( ( w * 2 ) / 4 ) * ( 5 / 7 ) , ( ( w * 3 ) / 4 ) * ( 5 / 7 ) , w * ( 5 / 7 ) , w * ( 5 / 7 + 1 / 21 ) , w * ( 5 / 7 + 2 / 21 ) , w * ( 6 / 7 ) , w * 1 , ] ; const segmentWidth = w / top . length ; const ctx = canvas . getContext ( '2d' ) ; for ( let i = 0 ; i < top . length ; i ++ ) { ctx . fillStyle = top [ i ] ; ctx . fillRect ( xShift + i * segmentWidth , 0 , segmentWidth , h1 ) ; ctx . fillStyle = middle [ i ] ; ctx . fillRect ( xShift + i * segmentWidth , h1 , segmentWidth , h2 - h1 ) ; } for ( let i = 0 ; i < bottom . length ; i ++ ) { ctx . fillStyle = bottom [ i ] ; ctx . fillRect ( xShift + bottomX [ i ] , h2 , bottomX [ i + 1 ] - bottomX [ i ] , h3 - h2 ) ; } } function makeColorBars ( canvas , colorOrPattern ) { const scheduler = new IntervalScheduler ( 1000 ) ; const context = canvas . getContext ( '2d' ) ; const stream = canvas . captureStream ( 5 ) || null ; if ( ! stream ) { return undefined ; } const onTick = ( ) : void => { if ( colorOrPattern === 'smpte' ) { fillSMPTEColorBars ( canvas , 0 ) ; } else { context . fillStyle = colorOrPattern ; context . fillRect ( 0 , 0 , canvas . width , canvas . height ) ; } } ; scheduler . start ( onTick ) ; const listener = ( ) : void => { scheduler . stop ( ) ; } ; stream . getVideoTracks ( ) [ 0 ] . addEventListener ( 'ended' , listener ) ; return { listener , scheduler , stream } ; } export default class DefaultDeviceController implements DeviceControllerBasedMediaStreamBroker , Destroyable { private static permissionDeniedOriginDetectionThresholdMs = 500 ; private static defaultVideoWidth = 960 ; private static defaultVideoHeight = 540 ; private static defaultVideoFrameRate = 15 ; private static defaultVideoMaxBandwidthKbps = 1400 ; private static defaultSampleRate = 48000 ; private static defaultSampleSize = 16 ; private static defaultChannelCount = 1 ; private static audioContext : AudioContext | null = null ; private deviceInfoCache : MediaDeviceInfo [ ] | null = null ; private transform : { nodes : AudioNodeSubgraph | undefined ; device : AudioTransformDevice ; } ; private activeDevices : { [ kind ] : DeviceSelection | null ; } = { audio : null , video : null } ; private chosenVideoTransformDevice : VideoTransformDevice | null = null ; private audioOutputDeviceId : string | null = null ; private deviceChangeObservers : Set < DeviceChangeObserver > = new Set < DeviceChangeObserver > ( ) ; private boundAudioVideoController : AudioVideoController | null ; private deviceLabelTrigger = ( ) : Promise < MediaStream > => { return navigator . mediaDevices . getUserMedia ( { audio : true , video : true } ) ; } ; private audioInputDestinationNode : MediaStreamAudioDestinationNode | null = null ; private audioInputSourceNode : MediaStreamAudioSourceNode | null = null ; private mediaDeviceWrapper : MediaDevices | undefined ; private onDeviceChangeCallback ? : Thunk ; private muteCallback : ( muted : boolean ) => void ; private videoInputQualitySettings : VideoQualitySettings = null ; private readonly useWebAudio : boolean = false ; private inputDeviceCount : number = 0 ; private lastNoVideoInputDeviceCount : number ; private mediaStreamMuteObserver = ( id , muted ) : void => { for ( const observer of this . deviceChangeObservers ) { AsyncScheduler . nextTick ( ( ) => { if ( this . deviceChangeObservers . has ( observer ) && observer . audioInputMuteStateChanged ) { observer . audioInputMuteStateChanged ( id , muted ) ; } } ) ; } } ; constructor ( private logger , options ? , private browserBehavior = new DefaultBrowserBehavior ( ) ) { const { enableWebAudio = false } = options || { } ; this . useWebAudio = enableWebAudio ; this . muteCallback = ( muted ) => { this . transform ?. device . mute ( muted ) ; } ; this . videoInputQualitySettings = new VideoQualitySettings ( DefaultDeviceController . defaultVideoWidth , DefaultDeviceController . defaultVideoHeight , DefaultDeviceController . defaultVideoFrameRate , DefaultDeviceController . defaultVideoMaxBandwidthKbps ) ; const dimension = this . browserBehavior . requiresResolutionAlignment ( this . videoInputQualitySettings . videoWidth , this . videoInputQualitySettings . videoHeight ) ; this . videoInputQualitySettings . videoWidth = dimension [ 0 ] ; this . videoInputQualitySettings . videoHeight = dimension [ 1 ] ; this . logger . info ( `DefaultDeviceController video dimension ${ this . videoInputQualitySettings . videoWidth } x ${ this . videoInputQualitySettings . videoHeight }` ) ; try { this . mediaDeviceWrapper = new DefaultMediaDeviceFactory ( ) . create ( ) ; const supportedConstraints = navigator . mediaDevices . getSupportedConstraints ( ) ; this . logger . info ( `Supported Constraints in this browser ${ JSON . stringify ( supportedConstraints ) }` ) ; } catch ( error ) { logger . error ( error . message ) ; } } private isWatchingForDeviceChanges ( ) { return ! ! this . onDeviceChangeCallback ; } private ensureWatchingDeviceChanges ( ) { if ( this . isWatchingForDeviceChanges ( ) ) { return ; } this . logger . info ( 'Starting devicechange listener.' ) ; this . onDeviceChangeCallback = ( ) => { this . logger . info ( 'Device change event callback is triggered' ) ; this . handleDeviceChange ( ) ; } ; this . mediaDeviceWrapper ?. addEventListener ( 'devicechange' , this . onDeviceChangeCallback ) ; } /**
     * Unsubscribe from the `devicechange` event, which allows the device controller to
     * update its device cache.
     */ private stopWatchingDeviceChanges ( ) { if ( ! this . isWatchingForDeviceChanges ( ) ) { return ; } this . logger . info ( 'Stopping devicechange listener.' ) ; this . mediaDeviceWrapper ?. removeEventListener ( 'devicechange' , this . onDeviceChangeCallback ) ; this . onDeviceChangeCallback = undefined ; } private shouldObserveDeviceChanges ( ) { if ( this . deviceChangeObservers . size ) { return true ; } const hasActiveDevices = ( this . activeDevices [ 'audio' ] && this . activeDevices [ 'audio' ] . constraints !== null ) || ( this . activeDevices [ 'video' ] && this . activeDevices [ 'video' ] . constraints !== null ) || ! ! this . audioOutputDeviceId ; return hasActiveDevices ; } private watchForDeviceChangesIfNecessary ( ) { if ( this . shouldObserveDeviceChanges ( ) ) { this . ensureWatchingDeviceChanges ( ) ; } else { this . stopWatchingDeviceChanges ( ) ; } } async destroy ( ) { this . stopWatchingDeviceChanges ( ) ; await this . chooseAudioInputDevice ( null ) ; await this . chooseVideoInputDevice ( null ) ; this . audioInputSourceNode ?. disconnect ( ) ; this . audioInputDestinationNode ?. disconnect ( ) ; this . audioInputSourceNode = undefined ; this . audioInputDestinationNode = undefined ; } async listAudioInputDevices ( forceUpdate = false ) { const result = await this . listDevicesOfKind ( 'audioinput' , forceUpdate ) ; this . trace ( 'listAudioInputDevices' , forceUpdate , result ) ; return result ; } async listVideoInputDevices ( forceUpdate = false ) { const result = await this . listDevicesOfKind ( 'videoinput' , forceUpdate ) ; this . trace ( 'listVideoInputDevices' , forceUpdate , result ) ; return result ; } async listAudioOutputDevices ( forceUpdate = false ) { const result = await this . listDevicesOfKind ( 'audiooutput' , forceUpdate ) ; this . trace ( 'listAudioOutputDevices' , forceUpdate , result ) ; return result ; } private pushAudioMeetingStateForPermissions ( device ) { this . boundAudioVideoController ?. eventController ?. publishEvent ( device === null ? 'audioInputUnselected' : 'audioInputSelected' ) ; } private pushVideoMeetingStateForPermissions ( device ) { this . boundAudioVideoController ?. eventController ?. publishEvent ( device === null ? 'videoInputUnselected' : 'videoInputSelected' ) ; } async chooseAudioInputDevice ( device ) { if ( device === undefined ) { this . logger . error ( 'Audio input device cannot be undefined' ) ; return ; } let recreateAudioContext = this . useWebAudio ; if ( ! this . useWebAudio ) { this . logger . debug ( 'Not using Web Audio. No need to recreate audio context.' ) ; } if ( DefaultDeviceController . audioContext ?. state === 'suspended' ) { recreateAudioContext = false ; } if ( recreateAudioContext && ! this . browserBehavior . requiresContextRecreationForAudioWorklet ( ) ) { this . logger . debug ( 'Browser does not require audio context recreation hack.' ) ; recreateAudioContext = false ; } if ( recreateAudioContext && ! this . transform && isAudioTransformDevice ( device ) ) { this . logger . debug ( 'Neither device is a transform. No need to recreate audio context.' ) ; recreateAudioContext = false ; } if ( recreateAudioContext ) { this . logger . info ( 'Recreating audio context when selecting new device.' ) ; if ( this . transform ) { if ( this . transform . nodes ) { this . transform . nodes . end . disconnect ( ) ; this . transform . nodes = undefined ; } this . transform = undefined ; } if ( this . audioInputSourceNode ) { this . audioInputSourceNode . disconnect ( ) ; this . audioInputSourceNode = undefined ; } if ( this . audioInputDestinationNode ) { this . audioInputDestinationNode . disconnect ( ) ; this . audioInputDestinationNode = undefined ; } DefaultDeviceController . closeAudioContext ( ) ; } if ( isAudioTransformDevice ( device ) ) { this . logger . info ( `Choosing transform input device ${ device }` ) ; await this . chooseAudioTransformInputDevice ( device ) ; } else { this . logger . info ( `Choosing intrinsic audio input device ${ device }` ) ; this . removeTransform ( ) ; await this . chooseInputIntrinsicDevice ( 'audio' , device , false ) ; this . trace ( 'chooseAudioInputDevice' , device , `success` ) ; } if ( recreateAudioContext && this . boundAudioVideoController ?. rtcPeerConnection ) { this . boundAudioVideoController . restartLocalAudio ( ( ) => { this . logger . info ( 'Local audio restarted.' ) ; } ) ; } this . pushAudioMeetingStateForPermissions ( device ) ; } private async chooseAudioTransformInputDevice ( device ) { if ( this . transform ?. device === device ) { return ; } if ( ! this . useWebAudio ) { throw new Error ( 'Cannot apply transform device without enabling Web Audio.' ) ; } const context = DefaultDeviceController . getAudioContext ( ) ; if ( context instanceof OfflineAudioContext ) { } else { switch ( context . state ) { case 'running' : break ; case 'closed' : throw new Error ( 'Cannot choose a transform device with a closed audio context.' ) ; case 'suspended' : await context . resume ( ) ; } } let nodes ; try { nodes = await device . createAudioNode ( context ) ; } catch ( e ) { this . logger . error ( `Unable to create transform device node: ${ e }.` ) ; throw e ; } const inner = await device . intrinsicDevice ( ) ; await this . chooseInputIntrinsicDevice ( 'audio' , inner , false ) ; this . logger . debug ( `Got inner stream: ${ inner }.` ) ; this . setTransform ( device , nodes ) ; } private async chooseVideoTransformInputDevice ( device ) { if ( device === this . chosenVideoTransformDevice ) { this . logger . info ( 'Reselecting same VideoTransformDevice' ) ; return ; } const prevVideoTransformDevice = this . chosenVideoTransformDevice ; if ( prevVideoTransformDevice ) { this . logger . info ( 'Switched from previous VideoTransformDevice' ) ; } const wasUsingTransformDevice = ! ! prevVideoTransformDevice ; const inner = await device . intrinsicDevice ( ) ; const canReuseMediaStream = this . isMediaStreamReusableByDeviceId ( this . activeDevices [ 'video' ] ?. stream , inner ) ; if ( ! canReuseMediaStream ) { this . logger . info ( 'video transform device needs new intrinsic device' ) ; if ( wasUsingTransformDevice ) { prevVideoTransformDevice . onOutputStreamDisconnect ( ) ; } this . chosenVideoTransformDevice = device ; this . activeDevices [ 'video' ] = null ; await this . chooseInputIntrinsicDevice ( 'video' , inner , false , true ) ; return ; } this . chosenVideoTransformDevice = device ; const newMediaStream = this . activeDevices [ 'video' ] . stream ; this . logger . info ( 'video transform device uses previous stream' ) ; if ( ! ( inner as MediaStream ) . id ) { const constraint = inner as MediaTrackConstraints ; constraint . width = constraint . width || this . videoInputQualitySettings . videoWidth ; constraint . height = constraint . height || this . videoInputQualitySettings . videoHeight ; constraint . frameRate = constraint . frameRate || this . videoInputQualitySettings . videoFrameRate ; await newMediaStream . getVideoTracks ( ) [ 0 ] . applyConstraints ( constraint ) ; } await device . transformStream ( this . activeDevices [ 'video' ] . stream ) ; if ( this . boundAudioVideoController ?. videoTileController . hasStartedLocalVideoTile ( ) ) { if ( this . boundAudioVideoController . replaceLocalVideo ) { this . restartLocalVideoAfterSelection ( false , true ) ; } else { await this . boundAudioVideoController . update ( { needsRenegotiation : true } ) ; } } } async chooseVideoInputDevice ( device ) { if ( device === undefined ) { this . logger . error ( 'Video input device cannot be undefined' ) ; return ; } if ( isVideoTransformDevice ( device ) ) { this . logger . info ( `Choosing video transform device ${ device }` ) ; return this . chooseVideoTransformInputDevice ( device ) ; } this . updateMaxBandwidthKbps ( ) ; if ( this . chosenVideoInputIsTransformDevice ( ) ) { this . chosenVideoTransformDevice . onOutputStreamDisconnect ( ) ; this . chosenVideoTransformDevice = null ; } await this . chooseInputIntrinsicDevice ( 'video' , device , false ) ; this . trace ( 'chooseVideoInputDevice' , device ) ; this . pushVideoMeetingStateForPermissions ( device ) ; } async chooseAudioOutputDevice ( deviceId ) { this . audioOutputDeviceId = deviceId ; this . watchForDeviceChangesIfNecessary ( ) ; await this . bindAudioOutput ( ) ; this . trace ( 'chooseAudioOutputDevice' , deviceId , null ) ; return ; } addDeviceChangeObserver ( observer ) { this . logger . info ( 'adding device change observer' ) ; this . deviceChangeObservers . add ( observer ) ; this . watchForDeviceChangesIfNecessary ( ) ; this . trace ( 'addDeviceChangeObserver' ) ; } removeDeviceChangeObserver ( observer ) { this . logger . info ( 'removing device change observer' ) ; this . deviceChangeObservers . delete ( observer ) ; this . watchForDeviceChangesIfNecessary ( ) ; this . trace ( 'removeDeviceChangeObserver' ) ; } createAnalyserNodeForAudioInput ( ) { if ( ! this . activeDevices [ 'audio' ] ) { return null ; } const node = this . transform ?. nodes ?. end ; if ( node ) { const analyser = node . context . createAnalyser ( ) as RemovableAnalyserNode ; analyser . removeOriginalInputs = ( ) => { try { node . disconnect ( analyser ) ; } catch ( e ) { } } ; node . connect ( analyser ) ; return analyser ; } return this . createAnalyserNodeForRawAudioInput ( ) ; } createAnalyserNodeForRawAudioInput ( ) { if ( ! this . activeDevices [ 'audio' ] ) { return null ; } return this . createAnalyserNodeForStream ( this . activeDevices [ 'audio' ] . stream ) ; } private createAnalyserNodeForStream ( stream ) { const audioContext = DefaultDeviceController . getAudioContext ( ) ; const analyser = audioContext . createAnalyser ( ) as RemovableAnalyserNode ; const source = audioContext . createMediaStreamSource ( stream ) ; source . connect ( analyser ) ; this . trace ( 'createAnalyserNodeForAudioInput' ) ; analyser . removeOriginalInputs = ( ) => { try { source . disconnect ( analyser ) ; } catch ( e ) { } } ; return analyser ; } startVideoPreviewForVideoInput ( element ) { if ( ! this . activeDevices [ 'video' ] ) { this . logger . warn ( 'cannot bind video preview since video input device has not been chosen' ) ; this . trace ( 'startVideoPreviewForVideoInput' , element . id ) ; return ; } const stream = element . srcObject as MediaStream ; if ( stream && this . activeDevices [ 'video' ] . stream !== stream ) { this . releaseMediaStream ( element . srcObject as MediaStream ) ; DefaultVideoTile . disconnectVideoStreamFromVideoElement ( element , false ) ; } if ( this . chosenVideoTransformDevice ) { DefaultVideoTile . connectVideoStreamToVideoElement ( this . chosenVideoTransformDevice . outputMediaStream , element , true ) ; } else { DefaultVideoTile . connectVideoStreamToVideoElement ( this . activeDevices [ 'video' ] . stream , element , true ) ; } this . trace ( 'startVideoPreviewForVideoInput' , element . id ) ; } stopVideoPreviewForVideoInput ( element ) { const stream = element . srcObject as MediaStream ; const activeVideoDevice = this . activeDevices [ 'video' ] ; if ( activeVideoDevice ) { this . releaseActiveDevice ( activeVideoDevice ) ; } if ( stream ) { this . releaseMediaStream ( stream ) ; DefaultVideoTile . disconnectVideoStreamFromVideoElement ( element , false ) ; } this . trace ( 'stopVideoPreviewForVideoInput' , element . id ) ; } setDeviceLabelTrigger ( trigger ) { if ( this . deviceInfoCache ) { for ( const device of this . deviceInfoCache ) { if ( ! device . label ) { this . deviceInfoCache = null ; break ; } } } this . deviceLabelTrigger = trigger ; this . trace ( 'setDeviceLabelTrigger' ) ; } mixIntoAudioInput ( stream ) { let node = null ; if ( this . useWebAudio ) { node = DefaultDeviceController . getAudioContext ( ) . createMediaStreamSource ( stream ) ; node . connect ( this . getMediaStreamOutputNode ( ) ) ; } else { this . logger . warn ( 'WebAudio is not enabled, mixIntoAudioInput will not work' ) ; } this . trace ( 'mixIntoAudioInput' , stream . id ) ; return node ; } chooseVideoInputQuality ( width , height , frameRate , maxBandwidthKbps ) { const dimension = this . browserBehavior . requiresResolutionAlignment ( width , height ) ; this . videoInputQualitySettings = new VideoQualitySettings ( dimension [ 0 ] , dimension [ 1 ] , frameRate , maxBandwidthKbps ) ; this . updateMaxBandwidthKbps ( ) ; } getVideoInputQualitySettings ( ) { return this . videoInputQualitySettings ; } acquireAudioInputStream ( ) { return this . acquireInputStream ( 'audio' ) ; } acquireVideoInputStream ( ) { return this . acquireInputStream ( 'video' ) ; } async acquireDisplayInputStream ( streamConstraints ) { if ( streamConstraints && streamConstraints . video && streamConstraints . video . mandatory && streamConstraints . video . mandatory . chromeMediaSource && streamConstraints . video . mandatory . chromeMediaSourceId ) { return navigator . mediaDevices . getUserMedia ( streamConstraints ) ; } return navigator . mediaDevices . getDisplayMedia ( streamConstraints ) ; } /**
     * This function helps `releaseMediaStream` do the right thing.
     *
     * We need to do three things:
     *
     * * Close the tracks of the source stream.
     * * Remove the transform.
     * * Clean up the _source_ stream's callback, as if `releaseMediaStream` had
     *   been called with that stream -- that's the stream that's tracked in
     *   `activeDevices` and needs to have its callbacks removed.
     *
     * This is a little fiddly because the stream broker interface doesn't
     * know about the innards of the device controller, and only has the
     * meeting session state's stream to work with.
     *
     */ private releaseAudioTransformStream ( ) { this . logger . info ( 'Stopping audio track for Web Audio graph' ) ; this . stopTracksAndRemoveCallbacks ( 'audio' ) ; this . logger . info ( 'Removing audio transform, if there is one.' ) ; this . removeTransform ( ) ; if ( this . audioInputSourceNode ) { this . audioInputSourceNode . disconnect ( ) ; this . audioInputSourceNode = undefined ; } if ( this . audioInputDestinationNode ) { this . audioInputDestinationNode . disconnect ( ) ; this . audioInputDestinationNode = undefined ; } } private releaseVideoTransformStream ( ) { this . logger . info ( 'Stopping video track for transform' ) ; this . stopTracksAndRemoveCallbacks ( 'video' ) ; this . logger . info ( 'Disconnecting video transform' ) ; this . chosenVideoTransformDevice . onOutputStreamDisconnect ( ) ; this . chosenVideoTransformDevice = null ; } private stopTracksAndRemoveCallbacks ( kind ) { const activeDevice = this . activeDevices [ kind ] ; if ( ! activeDevice ) { return ; } const endedCallback = activeDevice . endedCallback ; const trackMuteCallback = activeDevice . trackMuteCallback ; const trackUnmuteCallback = activeDevice . trackUnmuteCallback ; for ( const track of activeDevice . stream . getTracks ( ) ) { track . stop ( ) ; if ( endedCallback ) { track . removeEventListener ( 'ended' , endedCallback ) ; } if ( trackMuteCallback ) { track . removeEventListener ( 'mute' , trackMuteCallback ) ; } if ( trackUnmuteCallback ) { track . removeEventListener ( 'unmute' , trackUnmuteCallback ) ; } delete activeDevice . endedCallback ; delete activeDevice . trackMuteCallback ; delete activeDevice . trackUnmuteCallback ; delete this . activeDevices [ kind ] ; } } private releaseOrdinaryStream ( mediaStreamToRelease ) { const tracksToStop = mediaStreamToRelease . getTracks ( ) ; if ( ! tracksToStop . length ) { return ; } for ( const track of tracksToStop ) { track . stop ( ) ; } for ( const kind in this . activeDevices ) { const activeDevice = this . activeDevices [ kind ] ; if ( activeDevice ?. stream !== mediaStreamToRelease ) { continue ; } switch ( kind ) { case 'audio' : { for ( const track of mediaStreamToRelease . getAudioTracks ( ) ) { track . removeEventListener ( 'mute' , activeDevice . trackMuteCallback ) ; track . removeEventListener ( 'unmute' , activeDevice . trackUnmuteCallback ) ; track . removeEventListener ( 'ended' , activeDevice . endedCallback ) ; } delete activeDevice . trackMuteCallback ; delete activeDevice . trackUnmuteCallback ; delete activeDevice . endedCallback ; break ; } case 'video' : { const endedCallback = activeDevice . endedCallback ; if ( endedCallback ) { for ( const track of mediaStreamToRelease . getTracks ( ) ) { track . removeEventListener ( 'ended' , activeDevice . endedCallback ) ; } delete activeDevice . endedCallback ; } const tileController = this . boundAudioVideoController ?. videoTileController ; if ( tileController ?. hasStartedLocalVideoTile ( ) ) { tileController . stopLocalVideoTile ( ) ; } break ; } } delete this . activeDevices [ kind ] ; } } releaseMediaStream ( mediaStreamToRelease ) { if ( ! mediaStreamToRelease ) { return ; } try { const isReleasingAudioDestinationStream = mediaStreamToRelease === this . audioInputDestinationNode ?. stream ; if ( isReleasingAudioDestinationStream ) { this . releaseAudioTransformStream ( ) ; return ; } const isReleasingVideoOutputStream = mediaStreamToRelease === this . chosenVideoTransformDevice ?. outputMediaStream ; if ( isReleasingVideoOutputStream ) { this . releaseVideoTransformStream ( ) ; return ; } this . releaseOrdinaryStream ( mediaStreamToRelease ) ; } finally { this . watchForDeviceChangesIfNecessary ( ) ; } } private chosenVideoInputIsTransformDevice ( ) { return ! ! this . chosenVideoTransformDevice ; } bindToAudioVideoController ( audioVideoController ) { if ( this . boundAudioVideoController ) { this . unsubscribeFromMuteAndUnmuteLocalAudio ( ) ; } this . boundAudioVideoController = audioVideoController ; this . subscribeToMuteAndUnmuteLocalAudio ( ) ; if ( this . browserBehavior . supportsSetSinkId ( ) ) { AsyncScheduler . nextTick ( ( ) => { this . bindAudioOutput ( ) ; } ) ; } } private subscribeToMuteAndUnmuteLocalAudio ( ) { if ( ! this . boundAudioVideoController ) { return ; } if ( ! this . boundAudioVideoController . realtimeController ) { return ; } this . boundAudioVideoController . realtimeController . realtimeSubscribeToMuteAndUnmuteLocalAudio ( this . muteCallback ) ; } private unsubscribeFromMuteAndUnmuteLocalAudio ( ) { if ( ! this . boundAudioVideoController . realtimeController ) { return ; } this . boundAudioVideoController . realtimeController . realtimeUnsubscribeToMuteAndUnmuteLocalAudio ( this . muteCallback ) ; } static getIntrinsicDeviceId ( device ) { if ( device === undefined ) { return undefined ; } if ( device === null ) { return null ; } if ( typeof device === 'string' ) { return device ; } if ( ( device as MediaStream ) . id ) { return ( device as MediaStream ) . id ; } const constraints = device as MediaTrackConstraints ; const deviceIdConstraints = constraints . deviceId ; if ( deviceIdConstraints === undefined ) { return undefined ; } if ( deviceIdConstraints === null ) { return null ; } if ( typeof deviceIdConstraints === 'string' || Array . isArray ( deviceIdConstraints ) ) { return deviceIdConstraints ; } const constraintStringParams = deviceIdConstraints as ConstrainDOMStringParameters ; if ( typeof constraintStringParams . exact === 'string' || Array . isArray ( constraintStringParams . exact ) ) { return constraintStringParams . exact ; } return undefined ; } static createEmptyAudioDevice ( ) { return DefaultDeviceController . synthesizeAudioDevice ( 0 ) ; } static createEmptyVideoDevice ( ) { return DefaultDeviceController . synthesizeVideoDevice ( 'black' ) ; } static synthesizeAudioDevice ( toneHz ) { const audioContext = DefaultDeviceController . getAudioContext ( ) ; const outputNode = audioContext . createMediaStreamDestination ( ) ; if ( ! toneHz ) { const source = audioContext . createBufferSource ( ) ; try { source . buffer = audioContext . createBuffer ( 1 , audioContext . sampleRate * 5 , audioContext . sampleRate ) ; } catch ( error ) { if ( error && error . name === 'NotSupportedError' ) { source . buffer = audioContext . createBuffer ( 1 , DefaultDeviceController . defaultSampleRate * 5 , DefaultDeviceController . defaultSampleRate ) ; } else { throw error ; } } source . buffer . getChannelData ( 0 ) [ 0 ] = 0.0003 ; source . loop = true ; source . connect ( outputNode ) ; source . start ( ) ; } else { const gainNode = audioContext . createGain ( ) ; gainNode . gain . value = 0.1 ; gainNode . connect ( outputNode ) ; const oscillatorNode = audioContext . createOscillator ( ) ; oscillatorNode . frequency . value = toneHz ; oscillatorNode . connect ( gainNode ) ; oscillatorNode . start ( ) ; } return outputNode . stream ; } static synthesizeVideoDevice ( colorOrPattern ) { const canvas = document . createElement ( 'canvas' ) as HTMLCanvasElement ; canvas . width = 480 ; canvas . height = ( canvas . width / 16 ) * 9 ; const colorBars = makeColorBars ( canvas , colorOrPattern ) ; if ( ! colorBars ) { return null ; } const { stream } = colorBars ; return stream ; } private updateMaxBandwidthKbps ( ) { if ( this . boundAudioVideoController ) { this . boundAudioVideoController . setVideoMaxBandwidthKbps ( this . videoInputQualitySettings . videoMaxBandwidthKbps ) ; } } private async listDevicesOfKind ( deviceKind , forceUpdate ) { if ( forceUpdate || this . deviceInfoCache === null || ! this . isWatchingForDeviceChanges ( ) ) { await this . updateDeviceInfoCacheFromBrowser ( ) ; } return this . listCachedDevicesOfKind ( deviceKind ) ; } private async updateDeviceInfoCacheFromBrowser ( ) { const doesNotHaveAccessToMediaDevices = typeof MediaDeviceInfo === 'undefined' ; if ( doesNotHaveAccessToMediaDevices ) { this . deviceInfoCache = [ ] ; return ; } let devices = await navigator . mediaDevices . enumerateDevices ( ) ; let hasDeviceLabels = true ; for ( const device of devices ) { if ( ! device . label ) { hasDeviceLabels = false ; break ; } } if ( ! hasDeviceLabels ) { try { this . logger . info ( 'attempting to trigger media device labels since they are hidden' ) ; const triggerStream = await this . deviceLabelTrigger ( ) ; devices = await navigator . mediaDevices . enumerateDevices ( ) ; for ( const track of triggerStream . getTracks ( ) ) { track . stop ( ) ; } } catch ( err ) { this . logger . info ( 'unable to get media device labels' ) ; } } this . logger . debug ( `Update device info cache with devices: ${ JSON . stringify ( devices ) }` ) ; this . deviceInfoCache = devices ; } private listCachedDevicesOfKind ( deviceKind ) { const devicesOfKind = [ ] ; for ( const device of this . deviceInfoCache ) { if ( device . kind === deviceKind ) { devicesOfKind . push ( device ) ; } } return devicesOfKind ; } private alreadyHandlingDeviceChange = false ; private async handleDeviceChange ( ) { if ( this . deviceInfoCache === null ) { return ; } if ( this . alreadyHandlingDeviceChange ) { AsyncScheduler . nextTick ( ( ) => { this . handleDeviceChange ( ) ; } ) ; return ; } this . alreadyHandlingDeviceChange = true ; const oldAudioInputDevices = this . listCachedDevicesOfKind ( 'audioinput' ) ; const oldVideoInputDevices = this . listCachedDevicesOfKind ( 'videoinput' ) ; const oldAudioOutputDevices = this . listCachedDevicesOfKind ( 'audiooutput' ) ; await this . updateDeviceInfoCacheFromBrowser ( ) ; const newAudioInputDevices = this . listCachedDevicesOfKind ( 'audioinput' ) ; const newVideoInputDevices = this . listCachedDevicesOfKind ( 'videoinput' ) ; const newAudioOutputDevices = this . listCachedDevicesOfKind ( 'audiooutput' ) ; this . forEachObserver ( ( observer ) => { if ( ! this . areDeviceListsEqual ( oldAudioInputDevices , newAudioInputDevices ) ) { Maybe . of ( observer . audioInputsChanged ) . map ( f => f . bind ( observer ) ( newAudioInputDevices ) ) ; } if ( ! this . areDeviceListsEqual ( oldVideoInputDevices , newVideoInputDevices ) ) { Maybe . of ( observer . videoInputsChanged ) . map ( f => f . bind ( observer ) ( newVideoInputDevices ) ) ; } if ( ! this . areDeviceListsEqual ( oldAudioOutputDevices , newAudioOutputDevices ) ) { Maybe . of ( observer . audioOutputsChanged ) . map ( f => f . bind ( observer ) ( newAudioOutputDevices ) ) ; } } ) ; this . alreadyHandlingDeviceChange = false ; } private async handleDeviceStreamEnded ( kind , deviceId ) { try { await this . chooseInputIntrinsicDevice ( kind , null , false ) ; } catch ( e ) { this . logger . error ( 'Failed to choose null device after stream ended.' ) ; } if ( kind === 'audio' ) { this . forEachObserver ( ( observer ) => { Maybe . of ( observer . audioInputStreamEnded ) . map ( f => f . bind ( observer ) ( deviceId ) ) ; } ) ; } else { this . forEachObserver ( ( observer ) => { Maybe . of ( observer . videoInputStreamEnded ) . map ( f => f . bind ( observer ) ( deviceId ) ) ; } ) ; } } private forEachObserver ( observerFunc ) { for ( const observer of this . deviceChangeObservers ) { AsyncScheduler . nextTick ( ( ) => { if ( this . deviceChangeObservers . has ( observer ) ) { observerFunc ( observer ) ; } } ) ; } } private areDeviceListsEqual ( a , b ) { return ( JSON . stringify ( a . map ( device => JSON . stringify ( device ) ) . sort ( ) ) === JSON . stringify ( b . map ( device => JSON . stringify ( device ) ) . sort ( ) ) ) ; } private intrinsicDeviceAsMediaStream ( device ) { return device && device . id ? device : null ; } private hasSameMediaStreamId ( kind , selection , proposedConstraints ) { let streamId ; if ( kind === 'audio' ) { streamId = proposedConstraints ?. audio . streamId ; return ! ! streamId && streamId === selection . constraints ?. audio ?. streamId ; } streamId = proposedConstraints ?. video . streamId ; return ! ! streamId && streamId === selection ?. constraints ?. video ?. streamId ; } private hasSameGroupId ( groupId , kind , device ) { if ( groupId === '' ) { return true ; } const deviceIds = DefaultDeviceController . getIntrinsicDeviceId ( device ) ; this . logger . debug ( `Checking deviceIds ${ deviceIds } of type ${ typeof deviceIds } with groupId ${ groupId }` ) ; if ( typeof deviceIds === 'string' && groupId === this . getGroupIdFromDeviceId ( kind , deviceIds ) ) { return true ; } return false ; } private getGroupIdFromDeviceId ( kind , deviceId ) { if ( this . deviceInfoCache !== null ) { const cachedDeviceInfo = this . listCachedDevicesOfKind ( `${ kind }input` ) . find ( ( cachedDevice ) => cachedDevice . deviceId === deviceId ) ; if ( cachedDeviceInfo && cachedDeviceInfo . groupId ) { this . logger . debug ( `GroupId of deviceId ${ deviceId } found in cache is ${ cachedDeviceInfo . groupId }` ) ; return cachedDeviceInfo . groupId ; } } else { this . logger . error ( 'Device cache is not populated. Please make sure to call list devices first' ) ; } this . logger . debug ( `GroupId of deviceId ${ deviceId } found in cache is empty` ) ; return '' ; } private getActiveDeviceId ( kind ) { if ( this . activeDevices [ kind ] && this . activeDevices [ kind ] . constraints ) { const activeDeviceMediaTrackConstraints = this . activeDevices [ kind ] . constraints . audio || this . activeDevices [ kind ] . constraints . video ; const activeDeviceConstrainDOMStringParameters = ( activeDeviceMediaTrackConstraints as MediaTrackConstraints ) . deviceId ; let activeDeviceId ; if ( typeof activeDeviceConstrainDOMStringParameters === 'string' ) { activeDeviceId = activeDeviceConstrainDOMStringParameters as string ; } else { activeDeviceId = ( activeDeviceConstrainDOMStringParameters as ConstrainDOMStringParameters ) . exact as string ; } return activeDeviceId ; } return null ; } private async restartLocalVideoAfterSelection ( fromAcquire , fromVideoTransformDevice ) { if ( ! fromAcquire && this . boundAudioVideoController && this . boundAudioVideoController . videoTileController . hasStartedLocalVideoTile ( ) ) { if ( fromVideoTransformDevice ) { await this . boundAudioVideoController . replaceLocalVideo ( ) ; this . logger . info ( 'successfully replaced video track' ) ; } else { this . logger . info ( 'restarting local video to switch to new device' ) ; this . boundAudioVideoController . restartLocalVideo ( ( ) => { } ) ; } } } private handleGetUserMediaError ( error , errorTimeMs ? ) { if ( ! error ) { throw new GetUserMediaError ( error ) ; } switch ( error . name ) { case 'NotReadableError' : case 'TrackStartError' : throw new NotReadableError ( error ) ; case 'NotFoundError' : case 'DevicesNotFoundError' : throw new NotFoundError ( error ) ; case 'NotAllowedError' : case 'PermissionDeniedError' : case 'SecurityError' : if ( errorTimeMs && errorTimeMs < DefaultDeviceController . permissionDeniedOriginDetectionThresholdMs ) { throw new PermissionDeniedError ( error , 'Permission denied by browser' ) ; } else { throw new PermissionDeniedError ( error , 'Permission denied by user' ) ; } case 'OverconstrainedError' : case 'ConstraintNotSatisfiedError' : throw new OverconstrainedError ( error ) ; case 'TypeError' : throw new TypeError ( error ) ; case 'AbortError' : default : throw new GetUserMediaError ( error ) ; } } private releaseActiveDevice ( device ) { if ( ! device || ! device . stream ) { return ; } if ( device . endedCallback ) { const track = device . stream . getTracks ( ) [ 0 ] ; if ( track ) { track . removeEventListener ( 'ended' , device . endedCallback ) ; } } delete device . endedCallback ; this . releaseMediaStream ( device . stream ) ; delete device . stream ; } param kind param device param selection param proposedConstraints returns private matchesDeviceSelection ( kind , device , selection , proposedConstraints ) { if ( selection && selection . stream . active && ( this . hasSameMediaStreamId ( kind , selection , proposedConstraints ) || ( selection . groupId !== null && this . hasSameGroupId ( selection . groupId , kind , device ) ) ) ) { this . logger . debug ( `Compare current device constraint ${ JSON . stringify ( selection . constraints ) } to proposed constraints ${ JSON . stringify ( proposedConstraints ) }` ) ; return selection . matchesConstraints ( proposedConstraints ) ; } return false ; } private async chooseInputIntrinsicDevice ( kind , device , fromAcquire , fromVideoTransformDevice = false ) { this . inputDeviceCount += 1 ; const callCount = this . inputDeviceCount ; if ( device === null && kind === 'video' ) { this . lastNoVideoInputDeviceCount = this . inputDeviceCount ; const active = this . activeDevices [ kind ] ; if ( active ) { this . releaseActiveDevice ( active ) ; delete this . activeDevices [ kind ] ; this . watchForDeviceChangesIfNecessary ( ) ; } return ; } const proposedConstraints = this . calculateMediaStreamConstraints ( kind , device ) ; if ( this . matchesDeviceSelection ( kind , device , this . activeDevices [ kind ] , proposedConstraints ) ) { this . logger . info ( `reusing existing ${ kind } input device` ) ; return ; } if ( this . activeDevices [ kind ] && this . activeDevices [ kind ] . stream ) { if ( kind === 'audio' ) { this . releaseActiveDevice ( this . activeDevices [ kind ] ) ; } else if ( kind === 'video' ) { this . stopTracksAndRemoveCallbacks ( 'video' ) ; delete this . activeDevices [ kind ] ; } } const startTimeMs = Date . now ( ) ; const newDevice = new DeviceSelection ( ) ; try { this . logger . info ( `requesting new ${ kind } device with constraint ${ JSON . stringify ( proposedConstraints ) }` ) ; const stream = this . intrinsicDeviceAsMediaStream ( device ) ; if ( kind === 'audio' && device === null ) { newDevice . stream = DefaultDeviceController . createEmptyAudioDevice ( ) as MediaStream ; newDevice . constraints = null ; } else if ( stream ) { this . logger . info ( `using media stream ${ stream . id } for ${ kind } device` ) ; newDevice . stream = stream ; newDevice . constraints = proposedConstraints ; } else { newDevice . stream = await navigator . mediaDevices . getUserMedia ( proposedConstraints ) ; newDevice . constraints = proposedConstraints ; if ( kind === 'video' && this . lastNoVideoInputDeviceCount > callCount ) { this . logger . warn ( `ignored to get video device for constraints ${ JSON . stringify ( proposedConstraints ) } as no device was requested` ) ; this . releaseMediaStream ( newDevice . stream ) ; return ; } await this . handleDeviceChange ( ) ; const track = newDevice . stream . getTracks ( ) [ 0 ] ; newDevice . endedCallback = ( ) : void => { if ( this . activeDevices [ kind ] && this . activeDevices [ kind ] . stream === newDevice . stream ) { this . logger . warn ( `${ kind } input device which was active is no longer available, resetting to null device` ) ; this . handleDeviceStreamEnded ( kind , this . getActiveDeviceId ( kind ) ) ; delete newDevice . endedCallback ; } } ; track . addEventListener ( 'ended' , newDevice . endedCallback , { once : true } ) ; } if ( device !== null ) { const newDeviceId = this . getMediaTrackSettings ( newDevice . stream ) ?. deviceId ; newDevice . groupId = newDeviceId ? this . getGroupIdFromDeviceId ( kind , newDeviceId ) : '' ; } else { newDevice . groupId = '' ; } if ( kind === 'audio' ) { const track = newDevice . stream . getAudioTracks ( ) [ 0 ] ; if ( track ) { const id = track . getSettings ( ) . deviceId || newDevice . stream ; newDevice . trackMuteCallback = ( ) : void => { this . mediaStreamMuteObserver ( id , true ) ; } ; newDevice . trackUnmuteCallback = ( ) : void => { this . mediaStreamMuteObserver ( id , false ) ; } ; track . addEventListener ( 'mute' , newDevice . trackMuteCallback , { once : false } ) ; track . addEventListener ( 'unmute' , newDevice . trackUnmuteCallback , { once : false } ) ; } } } catch ( error ) { let errorMessage ; if ( error ?. name && error . message ) { errorMessage = `${ error . name }: ${ error . message }` ; } else if ( error ?. name ) { errorMessage = error . name ; } else if ( error ?. message ) { errorMessage = error . message ; } else { errorMessage = 'UnknownError' ; } if ( kind === 'audio' ) { this . boundAudioVideoController ?. eventController ?. publishEvent ( 'audioInputFailed' , { audioInputErrorMessage : errorMessage , } ) ; } else { this . boundAudioVideoController ?. eventController ?. publishEvent ( 'videoInputFailed' , { videoInputErrorMessage : errorMessage , } ) ; } this . logger . error ( `failed to get ${ kind } device for constraints ${ JSON . stringify ( proposedConstraints ) }: ${ errorMessage }` ) ; if ( error && 'constraint' in error ) { this . logger . error ( `Over-constrained by constraint: ${ error . constraint }` ) ; } if ( kind === 'audio' ) { this . logger . info ( `choosing null ${ kind } device instead` ) ; try { newDevice . stream = DefaultDeviceController . createEmptyAudioDevice ( ) as MediaStream ; newDevice . constraints = null ; await this . handleNewInputDevice ( kind , newDevice , fromAcquire ) ; } catch ( error ) { this . logger . error ( `failed to choose null ${ kind } device. ${ error . name }: ${ error . message }` ) ; } } this . handleGetUserMediaError ( error , Date . now ( ) - startTimeMs ) ; } finally { this . watchForDeviceChangesIfNecessary ( ) ; } this . logger . info ( `got ${ kind } device for constraints ${ JSON . stringify ( proposedConstraints ) }` ) ; await this . handleNewInputDevice ( kind , newDevice , fromAcquire , fromVideoTransformDevice ) ; if ( kind === 'audio' ) { this . logger . debug ( 'Notifying mute state after selection' ) ; for ( const track of newDevice . stream . getAudioTracks ( ) ) { if ( track . muted ) { newDevice . trackMuteCallback ( ) ; } else { newDevice . trackUnmuteCallback ( ) ; } } } return ; } private async handleNewInputDevice ( kind , newDevice , fromAcquire , fromVideoTransformDevice = false ) { this . activeDevices [ kind ] = newDevice ; this . logger . debug ( `Set activeDevice to ${ JSON . stringify ( newDevice ) }` ) ; this . watchForDeviceChangesIfNecessary ( ) ; if ( kind === 'video' ) { if ( this . chosenVideoInputIsTransformDevice ( ) ) { this . logger . info ( 'apply processors to transform' ) ; await this . chosenVideoTransformDevice . transformStream ( this . activeDevices [ 'video' ] . stream ) ; } await this . restartLocalVideoAfterSelection ( fromAcquire , fromVideoTransformDevice ) ; } else { if ( this . useWebAudio ) { this . attachAudioInputStreamToAudioContext ( this . activeDevices [ kind ] . stream ) ; } else if ( this . boundAudioVideoController ) { try { await this . boundAudioVideoController . restartLocalAudio ( ( ) => { } ) ; } catch ( error ) { this . logger . info ( `cannot replace audio track due to: ${ error . message }` ) ; } } else { this . logger . info ( 'no audio-video controller is bound to the device controller' ) ; } } } private async bindAudioOutput ( ) { if ( ! this . boundAudioVideoController ) { return ; } const deviceInfo = this . deviceInfoFromDeviceId ( 'audiooutput' , this . audioOutputDeviceId ) ; await this . boundAudioVideoController . audioMixController . bindAudioDevice ( deviceInfo ) ; } private calculateMediaStreamConstraints ( kind , device ) { let trackConstraints = { } ; if ( device === '' ) { device = null ; } const stream = this . intrinsicDeviceAsMediaStream ( device ) ; if ( device === null ) { return null ; } else if ( typeof device === 'string' ) { if ( this . browserBehavior . requiresNoExactMediaStreamConstraints ( ) && this . browserBehavior . requiresGroupIdMediaStreamConstraints ( ) ) { trackConstraints . deviceId = device ; trackConstraints . groupId = this . getGroupIdFromDeviceId ( kind , device ) ; } else if ( this . browserBehavior . requiresNoExactMediaStreamConstraints ( ) ) { trackConstraints . deviceId = device ; } else { trackConstraints . deviceId = { exact : device } ; } } else if ( stream ) { trackConstraints . streamId = stream . id ; } else if ( isMediaDeviceInfo ( device ) ) { trackConstraints . deviceId = device . deviceId ; trackConstraints . groupId = device . groupId ; } else { trackConstraints = device ; } if ( kind === 'video' ) { trackConstraints . width = trackConstraints . width || { ideal : this . videoInputQualitySettings . videoWidth , } ; trackConstraints . height = trackConstraints . height || { ideal : this . videoInputQualitySettings . videoHeight , } ; trackConstraints . frameRate = trackConstraints . frameRate || { ideal : this . videoInputQualitySettings . videoFrameRate , } ; } if ( kind === 'audio' && this . supportSampleRateConstraint ( ) ) { trackConstraints . sampleRate = { ideal : DefaultDeviceController . defaultSampleRate } ; } if ( kind === 'audio' && this . supportSampleSizeConstraint ( ) ) { trackConstraints . sampleSize = { ideal : DefaultDeviceController . defaultSampleSize } ; } if ( kind === 'audio' && this . supportChannelCountConstraint ( ) ) { trackConstraints . channelCount = { ideal : DefaultDeviceController . defaultChannelCount } ; } if ( kind === 'audio' ) { const augmented = { echoCancellation : true , googEchoCancellation : true , googEchoCancellation2 : true , googAutoGainControl : true , googAutoGainControl2 : true , googNoiseSuppression : true , googNoiseSuppression2 : true , googHighpassFilter : true , ... trackConstraints , } ; trackConstraints = augmented as MediaTrackConstraints ; } return kind === 'audio' ? { audio : trackConstraints } : { video : trackConstraints } ; } private deviceInfoFromDeviceId ( deviceKind , deviceId ) { if ( this . deviceInfoCache === null ) { return null ; } for ( const device of this . deviceInfoCache ) { if ( device . kind === deviceKind && device . deviceId === deviceId ) { return device ; } } return null ; } private async acquireInputStream ( kind ) { if ( kind === 'audio' ) { if ( this . useWebAudio ) { const dest = this . getMediaStreamDestinationNode ( ) ; return dest . stream ; } } if ( kind === 'video' ) { if ( this . chosenVideoInputIsTransformDevice ( ) ) { return this . chosenVideoTransformDevice . outputMediaStream ; } } let existingConstraints = null ; if ( ! this . activeDevices [ kind ] ) { if ( kind === 'audio' ) { this . logger . info ( `no ${ kind } device chosen, creating empty ${ kind } device` ) ; } else { this . logger . error ( `no ${ kind } device chosen, stopping local video tile` ) ; this . boundAudioVideoController . videoTileController . stopLocalVideoTile ( ) ; throw new Error ( `no ${ kind } device chosen, stopping local video tile` ) ; } } else { this . logger . info ( `checking whether existing ${ kind } input device can be reused` ) ; const active = this . activeDevices [ kind ] ; existingConstraints = active . constraints ? active . constraints [ kind ] : null ; } try { await this . chooseInputIntrinsicDevice ( kind , existingConstraints , true ) ; } catch ( e ) { this . logger . error ( `unable to acquire ${ kind } device` ) ; if ( e instanceof PermissionDeniedError ) { throw e ; } throw new GetUserMediaError ( e , `unable to acquire ${ kind } device` ) ; } return this . activeDevices [ kind ] . stream ; } hasAppliedTransform ( ) { return ! ! this . transform ; } private isMediaStreamReusableByDeviceId ( stream , device ) { if ( ! stream || ! stream . active || ! device ) { return false ; } if ( ( device as MediaStream ) . id ) { return stream . id === ( device as MediaStream ) . id ; } const settings = this . getMediaTrackSettings ( stream ) ; if ( ! settings . deviceId ) { return false ; } const deviceIds = DefaultDeviceController . getIntrinsicDeviceId ( device ) ; if ( typeof deviceIds === 'string' ) { return settings . deviceId === deviceIds ; } return false ; } private getMediaTrackSettings ( stream ) { return stream . getTracks ( ) [ 0 ] ?. getSettings ( ) ; } private reconnectAudioInputs ( ) { if ( ! this . audioInputSourceNode ) { return ; } this . audioInputSourceNode . disconnect ( ) ; const output = this . getMediaStreamOutputNode ( ) ; this . audioInputSourceNode . connect ( output ) ; } private setTransform ( device , nodes ) { this . transform ?. nodes ?. end . disconnect ( ) ; this . transform = { nodes , device } ; const proc = nodes ?. end ; const dest = this . getMediaStreamDestinationNode ( ) ; this . logger . debug ( `Connecting transform node ${ proc } to destination ${ dest }.` ) ; proc ?. connect ( dest ) ; this . reconnectAudioInputs ( ) ; } private removeTransform ( ) { const previous = this . transform ; if ( ! previous ) { return undefined ; } this . transform . nodes ?. end . disconnect ( ) ; this . transform = undefined ; this . reconnectAudioInputs ( ) ; return previous ; } private attachAudioInputStreamToAudioContext ( stream ) { this . audioInputSourceNode ?. disconnect ( ) ; this . audioInputSourceNode = DefaultDeviceController . getAudioContext ( ) . createMediaStreamSource ( stream ) ; const output = this . getMediaStreamOutputNode ( ) ; this . audioInputSourceNode . connect ( output ) ; } /**
     * Return the end of the Web Audio graph: post-transform audio.
     */ private getMediaStreamDestinationNode ( ) { if ( ! this . audioInputDestinationNode ) { this . audioInputDestinationNode = DefaultDeviceController . getAudioContext ( ) . createMediaStreamDestination ( ) ; } return this . audioInputDestinationNode ; } /**
     * Return the start of the Web Audio graph: pre-transform audio.
     * If there's no transform node, this is the destination node.
     */ private getMediaStreamOutputNode ( ) { return this . transform ?. nodes ?. start || this . getMediaStreamDestinationNode ( ) ; } static getAudioContext ( ) { if ( ! DefaultDeviceController . audioContext ) { const options = { } ; if ( navigator . mediaDevices . getSupportedConstraints ( ) . sampleRate ) { options . sampleRate = DefaultDeviceController . defaultSampleRate ; } DefaultDeviceController . audioContext = new ( window . AudioContext || window . webkitAudioContext ) ( options ) ; } return DefaultDeviceController . audioContext ; } static closeAudioContext ( ) { if ( DefaultDeviceController . audioContext ) { try { DefaultDeviceController . audioContext . close ( ) ; } catch ( e ) { } } DefaultDeviceController . audioContext = null ; } private supportSampleRateConstraint ( ) { return this . useWebAudio && ! ! navigator . mediaDevices . getSupportedConstraints ( ) . sampleRate ; } private supportSampleSizeConstraint ( ) { return this . useWebAudio && ! ! navigator . mediaDevices . getSupportedConstraints ( ) . sampleSize ; } private supportChannelCountConstraint ( ) { return this . useWebAudio && ! ! navigator . mediaDevices . getSupportedConstraints ( ) . channelCount ; } private trace ( name , input ? , output ? ) { let s = `API/DefaultDeviceController/${ name }` ; if ( typeof input !== 'undefined' ) { s += ` ${ JSON . stringify ( input ) }` ; } if ( typeof output !== 'undefined' ) { s += ` -> ${ JSON . stringify ( output ) }` ; } this . logger . info ( s ) ; } } function isMediaDeviceInfo ( device ) { return ( typeof device === 'object' && 'deviceId' in device && 'groupId' in device && 'kind' in device && 'label' in device ) ; }