import request from 'request-promise' ; import { GraphQLClient } from 'graphql-request' ; import { EventEmitter } from 'events' ; import { IPaginatedQueryOptions , IPaginatedQueryPageInfoData , IPaginatedQueryFilters } from '../interfaces/IPaginatedQuery' ; import log from './Logger' ; import * as Common from './Common' ; import SRQ from './StaggeredRequestQueue' ; import GQLClient from './GQLClient' ; import QueryQueue from './QueryQueue' ; import TokenHandler from './TokenHandler' ; import { merge , mergeQuery } from './Common' ; const RATE_LIMIT_MS_TIME = process . env . RateLimitMsTime || 1000 ; const TOTAL_PAGES_REGEX_JSON = new RegExp ( /"pageInfo":[\s]?{[\n\s]*?"totalPages": ([0-9]*)/ ) ; const TOTAL_PAGES_REGEX_STRING = new RegExp ( /"pageInfo":{"totalPages":([0-9]*)}/ ) ; const MAX_COMPLEXITY = 1000 ; export default class NetworkInterface { public static client : GraphQLClient ; public static initialized : boolean = false ; public static isClientDelinquent : boolean = false ; public static queryCount : number = 0 ; public static delinquencyTimer : any ; public static delinquencyQueue : any [ ] ; public static delinquencyPaginatedQueue : any [ ] ; public static init ( ) { if ( ! NetworkInterface . initialized ) { NetworkInterface . client = GQLClient . getInstance ( ) ; NetworkInterface . initialized = true ; } } param { string } query param { object } variables returns { promise } public static query ( query , variables ) { return new Promise ( ( resolve , reject ) => { log . queries ( 'Query: ' + JSON . stringify ( query ) + ':\n' + JSON . stringify ( variables ) ) ; QueryQueue . getInstance ( ) . add ( ( ) => { return NetworkInterface . client . request ( query , variables ) . then ( resolve ) . catch ( reject ) ; } ) ; } ) ; } public static rawQuery ( query , variables ) { return new Promise ( ( resolve , reject ) => { log . queries ( 'Raw Query: ' + JSON . stringify ( query ) + ':\n' + JSON . stringify ( variables ) ) ; QueryQueue . getInstance ( ) . add ( ( ) => { return NetworkInterface . client . rawRequest ( query , variables ) . then ( resolve ) . catch ( reject ) ; } ) ; } ) ; } public static staggeredQuery ( query , variables ) { return new Promise ( ( resolve , reject ) => { SRQ . getInstance ( ) . add ( ( ) => { return NetworkInterface . client . request ( query , variables ) . then ( resolve ) . catch ( reject ) ; } ) ; } ) ; } public static clusterQuery ( keys , fcn , options ) { return Promise . all ( keys . map ( key => { if ( ! key . hasOwnProperty ( fcn ) && ! key . __proto__ . hasOwnProperty ( fcn ) ) throw new Error ( `${ fcn } is not a function in type ${ typeof key }` ) ; return key [ fcn ] ( options ) ; } ) ) ; } public static async singleQuery ( query , variables ) { await Common . sleep ( + RATE_LIMIT_MS_TIME ) ; return await NetworkInterface . client . request ( query , variables ) ; } public static async paginatedQuery ( operationName , queryString , params , options ? , additionalParams ? , complexitySubtraction = 0 ) { log . info ( '%s: Calling Paginated Querys' , operationName ) ; log . queries ( 'Paginated Query: ' + JSON . stringify ( queryString ) + ':\n' + JSON . stringify ( params ) ) ; const results = [ ] ; const isSinglePage = options !== undefined && options . page ; const curPage = options !== undefined && options . page ? options . page : 1 ; const curFilters = options !== undefined && options . filters ? options . filters : null ; let curPerPage = options !== undefined && options . perPage ? options . perPage : null ; let queryOptions = { page : curPage , perPage : curPerPage , filters : curFilters , pageInfo : 'pageInfo{\ntotalPages\n}' } ; queryOptions = Object . assign ( queryOptions , additionalParams ) ; if ( isSinglePage ) { params = Object . assign ( params , queryOptions ) ; const singlePageQuery = mergeQuery ( queryString , queryOptions ) ; return [ await NetworkInterface . query ( singlePageQuery , params ) ] ; } const preflightQuery = mergeQuery ( queryString , queryOptions ) ; const preflightData = [ await NetworkInterface . rawQuery ( preflightQuery , params ) ] as any [ ] ; if ( preflightData . length <= 0 ) throw new Error ( `${ operationName }: No data returned from query for operation` ) ; const totalPages = NetworkInterface . parseTotalPages ( operationName , preflightData ) ; const onePageComplexity = preflightData [ 0 ] . extensions . queryComplexity - complexitySubtraction ; log . info ( 'Total Pages using 1 perPage: %s, Object Complexity per Page: %s' , totalPages , onePageComplexity ) ; curPerPage = NetworkInterface . calculateOptimalPerPagecount ( onePageComplexity , totalPages ) ; let query ; for ( let i = 1 ; i <= totalPages ; i ++ ) { log . info ( '%s: Collected %s/%s pages' , operationName , i , totalPages ) ; queryOptions = Object . assign ( { page : i , filters : curFilters , pageInfo : '' } , additionalParams , params ) ; query = mergeQuery ( queryString , queryOptions ) ; results . push ( await NetworkInterface . query ( query , queryOptions ) ) ; } return results ; } public static parseTotalPages ( operationName , results ) { const parsed = TOTAL_PAGES_REGEX_STRING . exec ( JSON . stringify ( results ) ) ; if ( ! parsed ) throw new Error ( `${ operationName }: Something wrong with paginated query. Did not match regex ${ TOTAL_PAGES_REGEX_STRING . toString ( ) }` ) ; return + parsed [ 1 ] ; } public static calculateOptimalPerPagecount ( objectComplexity , totalPages ) { const totalComplexity = objectComplexity * totalPages ; return MAX_COMPLEXITY / objectComplexity ; } public static determineComplexity ( objects ) { let complexity = 0 ; const nextArgs = [ ] ; for ( const i in objects ) { if ( i ) { complexity ++ ; const cur = objects [ i ] ; for ( const key in cur ) { if ( key === 'pageInfo' ) continue ; else if ( typeof cur [ key ] === 'object' && cur [ key ] != null ) { if ( Array . isArray ( cur [ key ] ) ) { complexity *= cur [ key ] . length ; nextArgs . push ( cur [ key ] [ 0 ] ) ; } else { nextArgs . push ( cur [ key ] ) ; } } } } } if ( nextArgs . length === 0 ) return complexity ; else return complexity + NetworkInterface . determineComplexity ( nextArgs ) ; } }