import { sum , mean } from 'd3-array' ; import { Page as PlaywrightPage , ChromiumBrowserContext } from 'playwright' ; import { Entry , RawResponse , EntryRequest , Page , EntryResponse , ResourceType } from './Protocol' ; import { AsyncQueue } from '../utils/AsyncQueue' ; import { Manager } from './Manager' ; import debugFactory from 'debug' ; const debug = debugFactory ( 'element:network:recorder' ) ; export type PageEvents = 'load' | 'close' | 'console' | 'crash' | 'dialog' | 'domcontentloaded' | 'download' | 'filechooser' | 'frameattached' | 'framedetached' | 'framenavigated' | 'pageerror' | 'popup' | 'request' | 'requestfailed' | 'requestfinished' | 'response' | 'worker' ; function round ( value ) { return Math . round ( value * 1000 ) / 1000 ; } function justNumber ( value , defaultValue ) { if ( value === undefined ) { return defaultValue ; } else { return value ; } } export default class Recorder { public entries : Entry [ ] ; private pages : Page [ ] ; public pendingTaskQueue : AsyncQueue < any > ; public manager : Manager ; constructor ( public page ) { this . reset ( ) ; this . pendingTaskQueue = new AsyncQueue ( ) ; this . manager = new Manager ( page ) ; } public toJSON ( ) { return { log : { version : '1.2' , creator : { name : 'Flood Chrome' , version : '1.0.0' } , pages : this . pages , entries : this . entries . map ( ( e ) => e . toJSON ( ) ) , } , } ; } public async sync ( ) { debug ( 'Recorder.sync() (pendingTaskQueue.chain)' ) ; await this . pendingTaskQueue . chain ; } public async recordRequest ( payload ) { debug ( 'Recorder.recordRequest(%o)' , payload ) ; if ( payload . request . url . startsWith ( 'data:' ) ) { return ; } const pageRef = payload . frameId ; const timestamp = payload . wallTime * 1e3 ; const basetime = timestamp - payload . timestamp * 1e3 ; const entry = new Entry ( { type : payload . type , requestId : payload . requestId , } ) ; entry . pageref = pageRef ; entry . request = new EntryRequest ( ) ; entry . request . setIssueTime ( payload . timestamp * 1e3 , payload . wallTime * 1e3 ) ; entry . request . method = payload . request . method ; entry . request . url = payload . request . url ; entry . request . timestamp = timestamp ; entry . request . _epoch = basetime ; entry . request . postData = '' ; entry . frameId = payload . frameId ; entry . loaderId = payload . loaderId ; this . entries . push ( entry ) ; if ( payload . type === 'Document' ) { this . recordPageResponse ( { _epoch : basetime , id : pageRef , startedDateTime : new Date ( timestamp ) , title : payload . documentURL , pageTimings : { onContentLoad : 0 , onLoad : 0 , } , } as Page ) ; } } public async recordResponse ( payload ) { const entry = this . getEntryForRequestId ( payload . requestId ) ; if ( ! entry ) return ; entry . type = payload . type ; entry . request . timing = payload . response . timing ; entry . request . endTime = payload . timestamp * 1e3 ; entry . response = new EntryResponse ( ) ; entry . response . timestamp = entry . request . pseudoWallTime ( payload . timestamp * 1e3 ) ; entry . serverIPAddress = payload . response . remoteIPAddress ; entry . connection = payload . response . connectionId . toString ( ) ; if ( payload . response . requestHeaders ) { entry . request . headers = Object . entries < string > ( payload . response . requestHeaders ) . map ( ( [ k , v ] ) => ( { name : k , value : v . toString ( ) , } ) ) ; } if ( payload . response && payload . response . requestHeadersText ) { entry . request . headersSize = payload . response . requestHeadersText . length ; } entry . request . httpVersion = payload . response . protocol ; entry . response . headers = Object . entries < string > ( payload . response . headers ) . map ( ( [ k , v ] ) => ( { name : k , value : v . toString ( ) , } ) ) ; if ( payload . response . headersText ) { entry . response . headersSize = payload . response . headersText . length ; } entry . response . content = { mimeType : payload . response . mimeType , text : '' , size : 0 } ; entry . response . bodySize = payload . response . encodedDataLength ; entry . response . httpVersion = payload . response . protocol ; entry . response . status = payload . response . status ; entry . response . statusText = payload . response . statusText ; } public async recordResponseCompleted ( { requestId , encodedDataLength , timestamp , } ) { debug ( `Recorder.recordResponseCompleted: ${ requestId }` ) ; const entry = this . getEntryForRequestId ( requestId ) ; if ( ! entry ) { return ; } const epoch = entry . request . _epoch ; entry . response . bodySize = encodedDataLength ; entry . response . timestamp = epoch + timestamp * 1e3 ; } param { string } documentType returns { Entry [ ] } memberof entriesForType ( documentType ) { return this . entries . filter ( ( { type } ) => type === documentType ) ; } readonly type { number } memberof public get documentResponseCode ( ) : number { const [ page ] = this . pages ; if ( page ) { const entries = this . entriesForPage ( page ) ; const entry = entries . find ( ( entry ) => String ( entry . type ) === 'Document' ) ; if ( entry ) return entry . response . status ; } return 0 ; } public entriesForPage ( page ) ; public entriesForPage ( pageId ) ; public entriesForPage ( page ) { if ( typeof page === 'string' ) { page = this . pages . find ( ( { id } ) => id === page ) ; } return this . entries . filter ( ( entry ) => entry . pageref === page . id ) ; } param { ResourceType } type return { number } public networkThroughputByType ( type ) { const entries = this . entries . map ( ( entry ) => entry . response . bodySize ) ; return sum ( entries ) ; } returns { number } memberof public networkThroughput ( ) { const entries = this . entries . map ( ( entry ) => entry . response . bodySize ) ; return round ( sum ( entries ) ) ; } public responseTime ( ) { return round ( sum ( this . entries . map ( ( { request } ) => request . duration ) ) ) ; } public meanResponseTime ( ) { return justNumber ( mean ( this . entries . map ( ( { request } ) => request . duration ) ) , 0 ) ; } public responseTimeForType ( type ) { const entries = this . entriesForType ( type ) . filter ( ( { request } ) => request . duration > 0 ) ; return round ( sum ( entries . map ( ( { request } ) => request . duration ) ) ) ; } public latencyForType ( type ) { const entries = this . entriesForType ( type ) . filter ( ( { request } ) => request . latency > 0 ) ; return round ( sum ( entries . map ( ( { request } ) => request . latency ) ) / entries . length ) ; } public timeToFirstByteForType ( type ) { const entries = this . entriesForType ( type ) . filter ( ( { request } ) => request . ttfb > 0 ) ; return round ( sum ( entries . map ( ( { request } ) => request . ttfb ) ) ) ; } public reset ( ) { debug ( 'Recorder.reset()' ) ; this . entries = [ ] ; this . pages = [ ] ; } public addPendingTask ( promise ) { this . pendingTaskQueue . add ( promise ) ; } public recordDOMContentLoadedEvent ( ) { } public async attachEvents ( ) { await this . manager . attachEvents ( ) ; } param { ( playwright . PageEvents ) } pageEvent param { ( event : any ) => void } handler public async attachEvent ( pageEvent , handler ) { if ( pageEvent . includes ( '.' ) ) { try { const browserContext = ( await this . page . context ( ) ) as ChromiumBrowserContext ; const client = await browserContext . newCDPSession ( this . page ) ; await client . send ( 'Network.enable' ) ; client . on ( pageEvent , handler ) ; } catch ( err ) { debug ( err ) ; } } else { this . page . on ( pageEvent , handler ) ; } } private recordPageResponse ( payload ) { this . pages . push ( payload ) ; } private getEntryForRequestId ( requestId ) { return this . entries . find ( ( entry ) => entry . requestId === requestId ) ; } private async privateClientSend ( method , ... args ) { const client = await ( this . page . context ( ) as ChromiumBrowserContext ) . newCDPSession ( this . page ) ; return client . send ( method , ... args ) ; } public async getResponseData ( requestId ) { debug ( `Recorder.getResponseData: ${ requestId }` ) ; try { const response = await this . privateClientSend ( 'Network.getResponseBody' , { requestId , } ) ; return Buffer . from ( response . body , response . base64Encoded ? 'base64' : 'utf8' ) ; } catch ( err ) { console . error ( `Recorder.getResponseData: ${ err . message }` ) ; return Buffer . from ( '' , 'utf8' ) ; } } }