nl_tokens: Parse a sitemap index document For each qualifying sitemap element download parse the sitemap . Stops processing if an error occurs .
pl_tokens: function parse ( options , document , callback ) { xml2js . parseString ( document , { trim : true , normalizeTags : true , normalize : true } , function ( err , result ) { var sitemapUrls = [ ] ; var sitemapIndexOptions = Object . assign ( { } , options , { outputPath : undefined } ) ; if ( ! err ) { // Check if we should process each sitemap in the index. _ . forEach ( // if the sitemap index is malformed, just blow up result . sitemapindex . sitemap , function ( sitemapNode ) { // optionally ignore current sitemaps by sitemap policy var shouldProcess = ! options . sitemapPolicy || ! smLib . stillCurrent ( sitemapNode , sitemapIndexOptions ) ; if ( shouldProcess ) { // if sitemap index is malformed, just blow up sitemapUrls . push ( sitemapNode . loc [ 0 ] ) ; } } ) ; // Edge case message for clarity if ( sitemapUrls . length === 0 ) { console . log ( "[*] No sitemaps qualified for processing" ) ; } // Get all the sitemaps, parse them, and generate input for each Promise . all ( sitemapUrls . map ( function ( sitemapUrl ) { var sitemapOptions = Object . assign ( { } , options , { source : sitemapUrl , input : "sitemap" , sitemapOutputDir : false , __sitemapIndex : sitemapIndexOptions } ) ; console . <mask> ( "[+] Loading the sitemap: '" + sitemapUrl + "'" ) ; return nodeCall ( smLib . getUrl , sitemapOptions , processSitemap ) ; } ) ) . then ( function ( ) { // Ignore the array of results to this function callback ( ) ; } ) . catch ( callback ) ; } else { callback ( err ) ; } } ) ; }