import { Browser } from 'puppeteer' ; import { extractPageContentAndUrls } from './core' ; import { CrawlDestination , CrawlInput } from './types' ; param browser param input param destination export const crawl = async ( browser , input , destination ? ) => { const pathQueue = [ ... input . startPaths ] ; const seenPaths = new Set ( pathQueue ) ; while ( pathQueue . length > 0 ) { const path = pathQueue . pop ( ) ! ; seenPaths . add ( path ) ; console . log ( 'Visiting' , path ) ; const newPaths = await extractPageContentAndUrls ( browser , { ... input , path , } , destination ) ; pathQueue . push ( ... newPaths . filter ( ( newPath ) => ! seenPaths . has ( newPath ) ) ) ; } return { visitedUrls : [ ... seenPaths ] } ; } ;