import AwaitLock from "await-lock" ; import chalk from "chalk" ; import { isLeft } from "fp-ts/lib/Either" ; import { Type } from "io-ts" ; import { PathReporter } from "io-ts/lib/PathReporter" ; import { ThrowReporter } from "io-ts/lib/ThrowReporter" ; import * as _ from "lodash/object" ; import { Browser , Headers , launch , LaunchOptions , Page , Request , Response , } from "puppeteer" ; import * as winston from "winston" ; import { AsyncPluginEventsType , IPlugin , IPluginContext , PluginEventsType , SyncPluginEvents , SyncPluginEventsType , } from "../../plugins" ; import { IOptions } from "./api" ; import { PostIdSet } from "./postIdSet" ; type AsyncPluginFunctions = { [ key in AsyncPluginEventsType ] : ( ( ... args : any [ ] ) => Promise < void > ) [ ] ; } ; type SyncPluginFunctions = { [ key in SyncPluginEventsType ] : ( ( ... args : any [ ] ) => void ) [ ] ; } ; type PluginFunctions = AsyncPluginFunctions & SyncPluginFunctions ; /**
 * Instagram API wrapper
 */ export class Instagram < PostType > { /**
     * Apply defaults to undefined options
     */ private static defaultOptions ( options ) { if ( options . enableGrafting === undefined ) { options . enableGrafting = true ; } if ( options . sameBrowser === undefined ) { options . sameBrowser = false ; } if ( options . fullAPI === undefined ) { options . fullAPI = false ; } if ( options . headless === undefined ) { options . headless = true ; } if ( options . logger === undefined ) { options . logger = winston . createLogger ( { silent : true , } ) ; } if ( options . silent === undefined ) { options . silent = true ; } if ( options . sleepTime === undefined ) { options . sleepTime = 2 ; } if ( options . hibernationTime === undefined ) { options . hibernationTime = 60 * 20 ; } if ( options . total === undefined ) { options . total = 0 ; } return options ; } public id : string ; public url : string ; public started : boolean = false ; public paused : boolean = false ; public finished : boolean = false ; public finishedReason : FinishedReasons ; public catchURL : string = "https://www.instagram.com/graphql/query" ; public postURL : string = "https://www.instagram.com/p/" ; public defaultPostURL : string = "https://www.instagram.com/p/" ; public jumpMod : number = 100 ; public jumpSize : number = 2 ; public page : Page ; public logger : winston . Logger ; public defaultPageFunctions : ( ( ) => void ) [ ] = [ ] ; private readonly strict : boolean = false ; private readonly validator : Type < unknown > ; private browser : Browser ; private browserDisconnected : boolean = true ; private readonly browserInstance ? : Browser ; private readonly headless : boolean ; private postBuffer : PostType [ ] = [ ] ; private postBufferLock : AwaitLock = new AwaitLock ( ) ; private requestBuffer : Request [ ] = [ ] ; private requestBufferLock : AwaitLock = new AwaitLock ( ) ; private responseBuffer : Response [ ] = [ ] ; private responseBufferLock : AwaitLock = new AwaitLock ( ) ; private readonly fullAPI : boolean = false ; private pagePromises : Promise < void > [ ] = [ ] ; private readonly enableGrafting : boolean = true ; private readonly sameBrowser : boolean = false ; private graft : boolean = false ; private graftURL : string = null ; private graftHeaders : Headers = null ; private foundGraft : boolean = false ; private hibernate : boolean = false ; private readonly hibernationTime : number = 60 * 20 ; private failedJumps : number = 20 ; private responseFromAPI : boolean = false ; private readonly pageQuery : string ; private readonly edgeQuery : string ; private postIds : PostIdSet ; private readonly total : number ; private index : number = 0 ; private jumps : number = 0 ; private readonly maxPageUrlAttempts = 3 ; private pageUrlAttempts = 0 ; private postPageRetries = 5 ; private readonly silent : boolean = false ; private writeLock : AwaitLock = new AwaitLock ( ) ; private sleepRemaining : number = 0 ; private readonly sleepTime : number = 2 ; private readonly proxyURL : string ; private readonly executablePath : string ; private pluginFunctions : PluginFunctions = { browser : [ ] , construction : [ ] , grafting : [ ] , postPage : [ ] , request : [ ] , response : [ ] , } ; param endpoint param id param pageQuery param edgeQuery param options param validator constructor ( endpoint , id , pageQuery , edgeQuery , options = { } , validator ) { this . id = id ; this . postIds = new PostIdSet ( ) ; this . url = endpoint . replace ( "[id]" , id ) ; options = Instagram . defaultOptions ( options ) ; this . total = options . total ; this . pageQuery = pageQuery ; this . edgeQuery = edgeQuery ; this . browserInstance = options . browserInstance ; this . headless = options . headless ; this . logger = options . logger ; this . silent = options . silent ; this . strict = options . strict ; this . enableGrafting = options . enableGrafting ; this . sameBrowser = options . sameBrowser ; this . sleepTime = options . sleepTime ; this . hibernationTime = options . hibernationTime ; this . fullAPI = options . fullAPI ; this . proxyURL = options . proxyURL ; this . executablePath = options . executablePath ; this . validator = options . validator || validator ; this . addPlugins ( options [ "plugins" ] ) ; this . executePlugins ( "construction" ) ; } /**
     * Toggle pausing data collection
     */ public pause ( ) { this . paused = ! this . paused ; } /**
     * Toggle prolonged pausing
     */ public toggleHibernation ( ) { this . hibernate = true ; } /**
     * Force the API to stop
     */ public async forceStop ( force ? ) { if ( ! force && ! this . started ) { return ; } this . started = false ; this . finish ( FinishedReasons . FORCED_STOP ) ; try { this . requestBufferLock . release ( ) ; } catch ( e ) { } try { this . responseBufferLock . release ( ) ; } catch ( e ) { } await this . stop ( ) ; } /**
     * Generator of posts on page
     */ public async * generator ( ) { if ( ! this . started ) { await this . start ( ) ; } while ( true ) { await this . getNext ( ) ; let post = await this . postPop ( ) ; while ( post ) { yield post ; post = await this . postPop ( ) ; } if ( this . finished && this . pagePromises . length === 0 ) { break ; } } await this . stop ( ) ; if ( ! this . silent ) { process . stdout . write ( "\n" ) ; } } /**
     * Construct page and add listeners
     */ public async start ( ) { let pageConstructed ; this . pageUrlAttempts = 0 ; while ( this . pageUrlAttempts ++ < this . maxPageUrlAttempts ) { pageConstructed = await this . constructPage ( ) ; if ( pageConstructed ) { break ; } } if ( ! pageConstructed ) { await this . forceStop ( true ) ; throw new Error ( "Failed to visit URL" ) ; } await this . executePlugins ( "browser" ) ; this . started = true ; await this . page . setRequestInterception ( true ) ; this . page . on ( "request" , ( req ) => this . interceptRequest ( req ) ) ; this . page . on ( "response" , ( res ) => this . interceptResponse ( res ) ) ; this . page . on ( "requestfailed" , ( res ) => this . interceptFailure ( res ) ) ; this . page . on ( "console" , ( message ) => this . logger . info ( "Console log" , { message } ) ) ; this . page . on ( "dialog" , ( dialog ) => dialog . dismiss ( ) ) ; this . page . on ( "error" , ( error ) => this . logger . error ( "Console error" , { error } ) ) ; if ( this . fullAPI ) { await this . scrapeDefaultPosts ( ) ; } } /**
     * Match the url to the url used in API requests
     */ public matchURL ( url ) { return url . startsWith ( this . catchURL ) && ! url . includes ( "include_reel" ) ; } /**
     * Close the page and browser
     */ protected async stop ( ) { await this . progress ( Progress . CLOSING ) ; if ( ! this . page . isClosed ( ) ) { this . page . removeAllListeners ( "request" ) ; this . page . removeAllListeners ( "response" ) ; this . page . removeAllListeners ( "requestfailed" ) ; } await this . requestBufferLock . acquireAsync ( ) ; this . requestBuffer = [ ] ; this . requestBufferLock . release ( ) ; await this . responseBufferLock . acquireAsync ( ) ; this . responseBuffer = [ ] ; this . responseBufferLock . release ( ) ; while ( true ) { if ( this . pagePromises . length === 0 ) { break ; } else { await this . sleep ( 1 ) ; } } if ( ! this . page . isClosed ( ) ) { await this . page . close ( ) ; } if ( ! this . browserDisconnected && ! this . browserInstance ) { await this . browser . close ( ) ; } } /**
     * Finish retrieving data for the generator
     */ protected finish ( reason ) { this . finished = true ; this . finishedReason = reason ; this . logger . info ( "Finished collecting" , { reason } ) ; } /**
     * Process the requests in the request buffer
     */ protected async processRequests ( ) { await this . requestBufferLock . acquireAsync ( ) ; let newApiRequest = false ; for ( const req of this . requestBuffer ) { if ( ! this . matchURL ( req . url ( ) ) ) { continue ; } else { newApiRequest = true ; } if ( this . graft ) { if ( this . foundGraft === false ) { this . graftURL = req . url ( ) ; this . graftHeaders = req . headers ( ) ; this . foundGraft = true ; await req . abort ( ) ; } else { const overrides = { headers : this . graftHeaders , url : this . graftURL , } ; await this . executePlugins ( "request" , req , overrides ) ; await req . continue ( overrides ) ; this . graft = false ; this . foundGraft = false ; this . graftURL = null ; this . graftHeaders = null ; } break ; } else { const overrides = { } ; this . executePlugins ( "request" , req , overrides ) ; await req . continue ( overrides ) ; } } this . requestBuffer = [ ] ; this . requestBufferLock . release ( ) ; if ( this . foundGraft && newApiRequest ) { await this . stop ( ) ; await this . start ( ) ; } } /**
     * Process the responses in the response buffer
     */ protected async processResponses ( ) { await this . responseBufferLock . acquireAsync ( ) ; for ( const res of this . responseBuffer ) { if ( ! this . matchURL ( res . url ( ) ) ) { continue ; } this . responseFromAPI = true ; let data ; try { data = await res . json ( ) ; if ( typeof data !== "object" ) { this . logger . error ( "Response data is not an object" , { data } ) ; continue ; } } catch ( error ) { this . logger . error ( "Error processing response JSON" , { data , error , } ) ; continue ; } this . executePlugins ( "response" , res , data ) ; if ( data && "status" in data && data [ "status" ] === "fail" ) { this . logger . info ( "Rate limited" ) ; this . hibernate = true ; continue ; } if ( ! ( _ . get ( data , this . pageQuery + ".has_next_page" , false ) && _ . get ( data , this . pageQuery + ".end_cursor" , false ) ) ) { this . logger . info ( "No posts remaining" , { data } ) ; this . finish ( FinishedReasons . API_FINISHED ) ; } await this . processResponseData ( data ) ; } this . responseBuffer = [ ] ; this . responseBufferLock . release ( ) ; } protected async processResponseData ( data ) { const posts = _ . get ( data , this . edgeQuery , [ ] ) ; for ( const post of posts ) { const postId = post [ "node" ] [ "id" ] ; const contains = this . postIds . add ( postId ) ; if ( contains ) { this . logger . info ( "Duplicate id found" , { postId } ) ; continue ; } if ( this . index < this . total || this . total === 0 ) { this . index ++ ; if ( this . fullAPI ) { this . pagePromises . push ( this . postPage ( post [ "node" ] [ "shortcode" ] , this . postPageRetries ) ) ; } else { await this . addToPostBuffer ( post ) ; } } else { this . finish ( FinishedReasons . TOTAL_REACHED_API ) ; break ; } } } /**
     * Open a post in a new page, then extract its metadata
     */ protected async postPage ( post , retries ) { const postPage = await this . browser . newPage ( ) ; await postPage . setRequestInterception ( true ) ; postPage . on ( "request" , async ( req ) => { if ( ! req . url ( ) . includes ( "/p/" + post ) ) { await req . abort ( ) ; } else { await req . continue ( ) ; } } ) ; postPage . on ( "requestfailed" , async ( req ) => this . interceptFailure ( req ) ) ; let parsed ; try { await postPage . goto ( this . postURL + post + "/" ) ; } catch ( error ) { await this . handlePostPageError ( postPage , error , "Couldn't navigate to page" , post , retries ) ; return ; } let data ; try { data = await postPage . evaluate ( async ( ) => { await new Promise ( ( resolve ) => { let i = 0 ; const findSharedData = setInterval ( ( ) => { if ( window [ "_sharedData" ] !== undefined || i ++ > 5 ) { resolve ( ) ; clearInterval ( findSharedData ) ; } } , 2000 ) ; } ) ; return JSON . stringify ( window [ "_sharedData" ] . entry_data . PostPage [ 0 ] . graphql ) ; } ) ; } catch ( error ) { await this . handlePostPageError ( postPage , error , "Couldn't evaluate on page" , post , retries ) ; return ; } await postPage . close ( ) ; try { parsed = JSON . parse ( data ) as PostType ; } catch ( error ) { await this . handlePostPageError ( postPage , error , "Couldn't parse page data" , post , retries ) ; return ; } await this . executePlugins ( "postPage" , parsed ) ; await this . addToPostBuffer ( parsed ) ; } private async handlePostPageError ( page , error , message , post , retries ) { this . logger . error ( message , { error } ) ; await this . progress ( Progress . ABORTED ) ; await this . sleep ( 2 ) ; if ( ! page . isClosed ( ) ) { await page . close ( ) ; } if ( retries > 0 ) { await this . postPage ( post , -- retries ) ; } } protected async validatePost ( post ) { const validationResult = this . validator . decode ( post ) ; if ( this . strict ) { ThrowReporter . report ( validationResult ) ; return ; } if ( isLeft ( validationResult ) ) { const validationReporter = PathReporter . report ( validationResult ) ; this . logger . warn ( `
      Warning! The Instagram API has been changed since this version of instamancer was released.
      More info: https://scriptsmith.github.io/instamancer/api-change
      ` , { validationReporter , post } ) ; } } /**
     * Stimulate the page until responses gathered
     */ protected async getNext ( ) { await this . progress ( Progress . SCRAPING ) ; while ( true ) { await this . processRequests ( ) ; await this . processResponses ( ) ; if ( this . pagePromises . length > 0 ) { await this . progress ( Progress . BRANCHING ) ; await Promise . all ( this . pagePromises ) ; this . pagePromises = [ ] ; } if ( this . finished ) { break ; } await this . waitResume ( ) ; await this . jump ( ) ; if ( this . jumps === this . failedJumps ) { if ( this . fullAPI ) { if ( ! this . responseFromAPI ) { this . finish ( FinishedReasons . NO_RESPONSE ) ; } } else if ( this . index === 0 ) { this . finish ( FinishedReasons . NO_INCREMENT ) ; const pageContent = { content : "" } ; try { pageContent . content = await this . page . content ( ) ; } catch ( e ) { } this . logger . error ( "Page failed to make requests" , pageContent ) ; break ; } } if ( this . jumps % this . jumpMod === 0 ) { await this . initiateGraft ( ) ; } await this . sleep ( this . sleepTime ) ; if ( this . hibernate ) { await this . sleep ( this . hibernationTime ) ; this . hibernate = false ; } await this . postBufferLock . acquireAsync ( ) ; const posts = this . postBuffer . length ; this . postBufferLock . release ( ) ; if ( posts > 0 ) { break ; } } } param time protected async sleep ( time ) { for ( let i = time ; i > 0 ; i -- ) { this . sleepRemaining = i ; await this . progress ( Progress . SCRAPING ) ; await new Promise ( ( resolve ) => { setTimeout ( resolve , i >= 1 ? 1000 : i * 1000 ) ; } ) ; } this . sleepRemaining = 0 ; await this . progress ( Progress . SCRAPING ) ; } /**
     * Create the browser and page, then visit the url
     */ private async constructPage ( ) { const args = [ ] ; if ( process . env . NO_SANDBOX ) { args . push ( "--no-sandbox" ) ; args . push ( "--disable-setuid-sandbox" ) ; } if ( this . proxyURL !== undefined ) { args . push ( "--proxy-server=" + this . proxyURL ) ; } const options = { args , headless : this . headless , } ; if ( this . executablePath !== undefined ) { options . executablePath = this . executablePath ; } if ( this . browserInstance ) { await this . progress ( Progress . LAUNCHING ) ; this . browser = this . browserInstance ; this . browserDisconnected = ! this . browser . isConnected ( ) ; this . browser . on ( "disconnected" , ( ) => ( this . browserDisconnected = true ) ) ; } else if ( ! this . sameBrowser || ( this . sameBrowser && ! this . started ) ) { await this . progress ( Progress . LAUNCHING ) ; this . browser = await launch ( options ) ; this . browserDisconnected = false ; this . browser . on ( "disconnected" , ( ) => ( this . browserDisconnected = true ) ) ; } this . page = await this . browser . newPage ( ) ; await this . progress ( Progress . OPENING ) ; try { await this . page . goto ( this . url ) ; const pageLoaded = await this . page . evaluate ( ( ) => { const headings = document . querySelectorAll ( "h2" ) ; for ( const heading of Array . from ( headings ) ) { if ( heading . innerHTML === "Sorry, this page isn't available." ) { return false ; } } return true ; } ) ; if ( ! pageLoaded ) { await this . handleConstructionError ( "Page loaded with no content" , 10 ) ; return false ; } for ( const f of this . defaultPageFunctions ) { await this . page . evaluate ( f ) ; } await this . page . evaluate ( ( ) => { setInterval ( ( ) => { try { document . body . style . overflow = "" ; } catch ( error ) { this . logger . error ( "Failed to update style" , { error } ) ; } } , 10000 ) ; } ) ; } catch ( e ) { await this . handleConstructionError ( e , 60 ) ; return false ; } return true ; } /***
     * Handle errors that occur during page construction
     */ private async handleConstructionError ( error , timeout ) { this . logger . error ( "Construction error" , { error , url : this . url } ) ; await this . progress ( Progress . ABORTED ) ; await this . sleep ( timeout ) ; if ( ! this . page . isClosed ( ) ) { await this . page . close ( ) ; } await this . browser . close ( ) ; } /**
     * Pause and wait until resumed
     */ private async waitResume ( ) { function f ( ) { return new Promise ( ( resolve ) => { setTimeout ( resolve , 200 ) ; } ) ; } while ( this . paused === true ) { await this . progress ( Progress . PAUSED ) ; await f ( ) ; } } /**
     * Pop a post off the postBuffer (using locks). Returns null if no posts in buffer
     */ private async postPop ( ) { let post = null ; await this . postBufferLock . acquireAsync ( ) ; if ( this . postBuffer . length > 0 ) { post = this . postBuffer . shift ( ) ; } this . postBufferLock . release ( ) ; return post ; } /**
     * Print progress to stderr
     */ private async progress ( state ) { if ( this . silent ) { return ; } await this . writeLock . acquireAsync ( ) ; const total = this . total === 0 ? "Unlimited" : this . total ; const idStr = chalk . bgYellow . black ( ` ${ this . id } ` ) ; const totalStr = chalk . bgBlack ( ` Total: ${ total } ` ) ; const stateStr = chalk . bgWhite . black ( ` State: ${ state } ` ) ; const sleepStr = chalk . bgWhite . black ( ` Sleeping: ${ this . sleepRemaining } ` ) ; const indexStr = chalk . bgWhite . black ( ` Scraped: ${ this . index } ` ) ; this . logger . debug ( { id : this . id , index : this . index , sleepRemaining : this . sleepRemaining , state , total , } ) ; process . stderr . write ( `\r${ idStr }${ totalStr }${ stateStr }${ sleepStr }${ indexStr }\u001B[K` ) ; this . writeLock . release ( ) ; } /**
     * Add request to the request buffer
     */ private async interceptRequest ( req ) { await this . requestBufferLock . acquireAsync ( ) ; this . requestBuffer . push ( req ) ; await this . requestBufferLock . release ( ) ; } /**
     * Add the response to the response buffer
     */ private async interceptResponse ( res ) { await this . responseBufferLock . acquireAsync ( ) ; this . responseBuffer . push ( res ) ; await this . responseBufferLock . release ( ) ; } /**
     * Log failed requests
     */ private async interceptFailure ( req ) { this . logger . info ( "Failed request" , { url : req . url ( ) } ) ; await this . progress ( Progress . ABORTED ) ; } /**
     * Add post to buffer
     */ private async addToPostBuffer ( post ) { await this . postBufferLock . acquireAsync ( ) ; await this . validatePost ( post ) ; this . postBuffer . push ( post ) ; this . postBufferLock . release ( ) ; } /**
     * Manipulate the page to stimulate a request
     */ private async jump ( ) { await this . page . keyboard . press ( "PageUp" ) ; const jumpSize = this . graft ? 1 : this . jumpSize ; for ( let i = 0 ; i < jumpSize ; i ++ ) { await this . page . keyboard . press ( "End" ) ; } const width = this . page . viewport ( ) [ "width" ] ; const height = this . page . viewport ( ) [ "height" ] ; await this . page . mouse . move ( Math . round ( width * Math . random ( ) ) , Math . round ( height * Math . random ( ) ) ) ; ++ this . jumps ; } /**
     * Clear request and response buffers
     */ private async initiateGraft ( ) { if ( ! this . enableGrafting ) { return ; } await this . progress ( Progress . GRAFTING ) ; this . executePlugins ( "grafting" ) ; this . graft = true ; } /**
     * Read the posts that are pre-loaded on the page
     */ private async scrapeDefaultPosts ( ) { const shortCodes = await this . page . evaluate ( ( url ) => { return Array . from ( document . links ) . filter ( ( link ) => { return ( link . href . startsWith ( url ) && link . href . split ( "/" ) . length >= 2 ) ; } ) . map ( ( link ) => { const linkSplit = link . href . split ( "/" ) ; return linkSplit [ linkSplit . length - 2 ] ; } ) ; } , this . defaultPostURL ) ; for ( const shortCode of shortCodes ) { if ( this . index < this . total || this . total === 0 ) { this . index ++ ; this . pagePromises . push ( this . postPage ( shortCode , this . postPageRetries ) ) ; } else { this . finish ( FinishedReasons . TOTAL_REACHED_PAGE ) ; break ; } } } private addPlugins ( plugins ) { if ( ! plugins ) { return ; } for ( const plugin of plugins ) { for ( const event of Object . keys ( this . pluginFunctions ) ) { const pluginEvent = plugin [ event + "Event" ] ; if ( pluginEvent ) { const context = { plugin , state : this , } ; this . pluginFunctions [ event ] . push ( pluginEvent . bind ( context ) ) ; } } } } private executePlugins ( event , ... args ) ; private executePlugins ( event , ... args ) ; private executePlugins ( event , ... args ) { if ( event in SyncPluginEvents ) { for ( const pluginFunction of this . pluginFunctions [ "construction" ] ) { pluginFunction ( ) ; } return ; } return Promise . all ( this . pluginFunctions [ event ] . map ( ( cb ) => cb ( ... args ) ) ) ; } } /**
 * The states of progress that the API can be in. Used to output status.
 */ enum Progress { LAUNCHING = "Launching" , OPENING = "Navigating" , SCRAPING = "Scraping" , BRANCHING = "Branching" , GRAFTING = "Grafting" , CLOSING = "Closing" , PAUSED = "Paused" , ABORTED = "Request aborted" } /**
 * Reasons why the collection finished
 */ enum FinishedReasons { FORCED_STOP , API_FINISHED , TOTAL_REACHED_API , TOTAL_REACHED_PAGE , NO_RESPONSE , NO_INCREMENT }