import * as AWS from 'aws-sdk' ; import * as chrome from 'chrome-aws-lambda' ; import { extractPageContentAndUrls } from '../crawler/core' ; import { CrawlContext } from '../crawler/types' ; import { markPathAsVisited , queuePaths } from '../utils/contextTable' ; const s3 = new AWS . S3 ( ) ; /**
 * This step is the main part of the webcrawler, responsible for extracting content from a single webpage, and adding
 * any newly discovered urls to visit to the queue.
 */ export const crawlPageAndQueueUrls = async ( path , crawlContext , dataSourceBucketName ? ) => { let browser ; try { const { contextTableName , baseUrl , pathKeywords , crawlName } = crawlContext ; await markPathAsVisited ( contextTableName , path ) ; console . log ( 'Marked path' , path , 'as visited.' ) ; browser = await chrome . puppeteer . launch ( { args : chrome . args , defaultViewport : chrome . defaultViewport , executablePath : await chrome . executablePath , headless : chrome . headless , ignoreHTTPSErrors : true , } ) ; const destination = dataSourceBucketName ? { s3 , s3BucketName : dataSourceBucketName , s3KeyPrefix : crawlName , } : undefined ; const urlPaths = await extractPageContentAndUrls ( browser , { baseUrl , path , pathKeywords , } , destination ) ; console . log ( 'Synced content from' , path ) ; console . log ( 'Queueing' , urlPaths . length , 'new urls to visit' , urlPaths ) ; await queuePaths ( contextTableName , urlPaths ) ; } catch ( e ) { console . error ( 'Failed to crawl path' , path , e ) ; } finally { browser && await browser . close ( ) ; } return { } ; } ;