import { Browser , Page } from 'puppeteer-core' ; import { URL } from 'url' ; import * as path from 'path' ; import { CrawlDestination , CrawlPageInput , PageContent } from './types' ; param page const extractContent = async ( page ) : Promise < PageContent > => { const [ title , htmlContent ] = await Promise . all ( [ page . evaluate ( ( ) => document . title ) , page . evaluate ( ( ) => document . body . innerHTML ) , ] ) ; return { title , htmlContent } ; } ; /**
 * Writes the given page content to S3, along with metadata for Kendra
 */ const writePageToS3ForKendra = async ( url , content , destination ) => { if ( ! content . title || ! content . htmlContent ) { console . log ( 'Page has no content, skipping' ) ; return ; } const documentKey = path . join ( destination . s3KeyPrefix , `${ encodeURIComponent ( url ) }.html` ) ; const metadata = { Title : content . title , Attributes : { _source_uri : url , } , } ; const s3Put = ( Key , Body ) => destination . s3 . putObject ( { Bucket : destination . s3BucketName , Key , Body } ) . promise ( ) ; await Promise . all ( [ s3Put ( documentKey , content . htmlContent ) , s3Put ( `${ documentKey }.metadata.json` , JSON . stringify ( metadata ) ) , ] ) ; console . log ( "Written page content to s3" ) ; } ; /**
 * Return whether the given url is within the website of the base url, ie it's a relative link, or it's an absolute
 * link that starts with the base url.
 */ const isUrlWithinBaseWebsite = ( url , baseUrl ) : boolean => ! url . startsWith ( 'http' ) || url . startsWith ( baseUrl ) ; /**
 * Return whether any of the keywords are included in the url. Keywords are optional, we include the url by default
 * if they aren't supplied.
 */ const isUrlMatchingSomeKeyword = ( url , keywords ? ) : boolean => ( ! keywords || keywords . length === 0 || keywords . some ( ( keyword ) => url . toLowerCase ( ) . includes ( keyword ) ) ) ; return const getLinksToFollow = async ( page , baseUrl , keywords ? ) : Promise < string [ ] > => { const urls = await page . $$eval ( 'a' , ( elements => elements . map ( e => e . getAttribute ( 'href' ) ) ) ) ; const currentPageUrlParts = ( await page . evaluate ( ( ) => document . location . href ) ) . split ( '/' ) ; const relativeUrlBase = currentPageUrlParts . slice ( 0 , currentPageUrlParts . length ) . join ( '/' ) ; return urls . filter ( ( url ) => url && isUrlWithinBaseWebsite ( url , baseUrl ) ) . map ( ( url ) => { if ( url ! . startsWith ( baseUrl ) ) { return url ! ; } const u = new URL ( url ! , relativeUrlBase ) ; return `${ u . origin }${ u . pathname }` ; } ) . filter ( ( url ) => isUrlMatchingSomeKeyword ( url , keywords ) ) ; } ; param browser param input param destination return export const extractPageContentAndUrls = async ( browser , input , destination ? ) : Promise < string [ ] > => { const url = new URL ( input . path , input . baseUrl ) . href ; try { const page = await browser . newPage ( ) ; await page . goto ( url , { waitUntil : 'networkidle0' , } ) ; const content = await extractContent ( page ) ; console . log ( "Extracted content from page:" , content ) ; if ( destination ) { await writePageToS3ForKendra ( url , content , destination ) ; } const discoveredUrls = new Set ( await getLinksToFollow ( page , input . baseUrl , input . pathKeywords ) ) ; console . log ( "Discovered urls:" , discoveredUrls ) ; const discoveredPaths = [ ... discoveredUrls ] . flatMap ( ( u ) => { try { return [ new URL ( u ) . pathname ] ; } catch ( e ) { console . warn ( 'Url' , u , 'was not valid and will be skipped' , e ) ; return [ ] ; } } ) ; console . log ( "Discovered relative paths:" , discoveredPaths ) ; return discoveredPaths ; } catch ( e ) { console . warn ( 'Could not visit url' , url , e ) ; return [ ] ; } } ;