import * as charset from "charset" ; import * as cheerio from "cheerio" ; import * as got from "got" ; import * as http from "http" ; import * as iconv from "iconv-lite" ; import * as url from "url" ; import { IPlan , ITask } from "../interfaces" ; import Spider from "../spider" ; import NodeSpider from "../spider" ; import requestPlan from "./requestPlan" ; import { ICurrent } from "./requestPlan" ; export type IHandle = ( $ : IJq , current : ICurrent , spider : Spider ) => any | Promise < any > ; export interface IOption { name : string ; handle : IHandle ; failed ? : ( error : Error , task : ITask , spider : Spider ) => any ; retries ? : number ; toUtf8 ? : boolean ; requestOpts ? : http . RequestOptions & { encoding : null ; } ; } export default function jqPlan ( option ) { return requestPlan ( { name : option . name , failed : option . failed , retries : option . retries , toUtf8 : option . toUtf8 , requestOpts : option . requestOpts , handle : async ( current , spider ) => { const $ = loadJq ( current ) ; await option . handle ( $ , current , spider ) ; } , } ) ; } export interface IJq extends CheerioStatic { ( selector ) : IJq ; urls : ( ) => string [ ] ; } /**
 * 根据currentTask.body加载jQ对象，并扩展url、todo、download方法，以第三个参数$的形式传递
 */ export function loadJq ( currentTask ) { const $ = cheerio . load ( currentTask . body ) ; $ . prototype . urls = function ( ) : string [ ] { const result = [ ] ; $ ( this ) . map ( ( ix , ele ) => { let newUrl = $ ( ele ) . attr ( "href" ) ; if ( ! newUrl || /^javascript:/ . test ( newUrl ) ) { return false ; } if ( newUrl && ! /^https?:\/\// . test ( newUrl ) ) { newUrl = url . resolve ( currentTask . url , newUrl ) ; } const anchorIndex = newUrl . lastIndexOf ( "#" ) ; if ( anchorIndex !== - 1 ) { newUrl = newUrl . slice ( 0 , anchorIndex ) ; } result . push ( newUrl ) ; } ) ; return result ; } ; return $ as IJq ; }