package com.wongsir.newsgathering.gather.commons;

import java.io.File;
import java.io.IOException;
import java.net.BindException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.UUID;
import java.util.stream.Collectors;

import javax.management.JMException;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.springframework.beans.factory.annotation.Autowired;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.wongsir.newsgathering.dao.CommonWebpageDAO;
import com.wongsir.newsgathering.dao.CommonWebpagePipeline;
import com.wongsir.newsgathering.dao.SpiderInfoDAO;
import com.wongsir.newsgathering.gather.async.AsyncGather;
import com.wongsir.newsgathering.gather.async.TaskManager;
import com.wongsir.newsgathering.model.async.State;
import com.wongsir.newsgathering.model.async.Task;
import com.wongsir.newsgathering.model.commons.SpiderInfo;
import com.wongsir.newsgathering.model.commons.Webpage;
import com.wongsir.newsgathering.utils.NLPExtractor;
import com.wongsir.newsgathering.utils.StaticValue;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.UrlUtils;

/** 
* @Description: TODO
* @author Wongsir
* @date 2017年1月4日
*  
*/
public class CommonSpider extends AsyncGather {
    private static final Logger LOG = LogManager.getLogger(CommonSpider.class);
    private static final String LINK_KEY = "LINK_LIST";
    private static final String DYNAMIC_FIELD = "dynamic_fields";
    private static final String SPIDER_INFO = "spiderInfo";
    private static List<String> ignoredUrls;
    //尽量先匹配长模板
    private static LinkedList<Pair<String, SimpleDateFormat>> datePattern = Lists.newLinkedList();

    static {
        try {
            ignoredUrls = FileUtils.readLines(new File(CommonSpider.class.getClassLoader().getResource("ignoredUrls.txt").getFile()));
            LOG.info("加载普通网页爬虫url忽略名单成功,忽略名单:{}", ignoredUrls);
            try {
                String[] datePatternFile = FileUtils.readFileToString(
                        new File(CommonSpider.class.getClassLoader().getResource("datePattern.txt").getFile()),
                        "utf8"
                ).replace("\r", "").split("=====\n");
                String[] dateList = datePatternFile[0].split("\n");
                String[] timeList = datePatternFile[1].split("\n");
                for (String date : dateList) {
                    String[] dateEntry = date.split("##");
                    String dateReg = dateEntry[0];
                    String dateFormat = dateEntry[1];
                    LOG.debug("正在编译日期正则{},format:{}", dateReg, dateFormat);
                    datePattern.add(Pair.of(dateReg, new SimpleDateFormat(dateFormat)));
                    for (String time : timeList) {
                        String[] timeEntry = time.split("##");
                        String timeReg = timeEntry[0];
                        String timeFormat = timeEntry[1];
                        //日期与时间中间有空格
                        LOG.debug("正在编译日期正则{},format:{}", dateReg + " " + timeReg, dateFormat + " " + timeFormat);
                        datePattern.add(Pair.of(dateReg + " " + timeReg, new SimpleDateFormat(dateFormat + " " + timeFormat)));
                        //日期与时间中间无空格
                        LOG.debug("正在编译日期正则{},format:{}", dateReg + timeReg, dateFormat + timeFormat);
                        datePattern.add(Pair.of(dateReg + timeReg, new SimpleDateFormat(dateFormat + timeFormat)));
                    }
                }
                datePattern.sort((o1, o2) -> o2.getLeft().length() - o1.getLeft().length());
                LOG.info("日期匹配式加载完成");
            } catch (IOException e) {
                LOG.error("加载日期匹配式失败，{}", e.getLocalizedMessage());
            }
        } catch (IOException e) {
            e.printStackTrace();
            LOG.error("加载普通网页爬虫url忽略名单失败", e);
        }
    }

    //慎用爬虫监控,可能导致内存泄露
    private SpiderMonitor spiderMonitor = SpiderMonitor.instance();
    private Map<String, MySpider> spiderMap = new HashMap<>();
    private NLPExtractor keywordsExtractor;
    private NLPExtractor summaryExtractor;
    private NLPExtractor namedEntitiesExtractor;
    private StaticValue staticValue;
    @SuppressWarnings("unchecked")
    private final PageConsumer spiderInfoPageConsumer = (page, info, task) -> {
        long start = System.currentTimeMillis();
//        if (page.getHtml().getDocument().getElementsByTag("a").size() <= 0) {
//            page.setSkip(true);
//            return;
//        }
        //本页是否是startUrls里面的页面
        final boolean startPage = info.getStartURL().contains(page.getUrl().get());
        List<String> attachmentList = Lists.newLinkedList();
        //判断本网站是否只抽取入口页,和当前页面是不是入口页
        if (!info.isGatherFirstPage() || (info.isGatherFirstPage() && startPage)) {
            List<String> links = null;
            if (StringUtils.isNotBlank(info.getUrlReg())) {//url正则式不为空
                links = page.getHtml().links().regex(info.getUrlReg()).all().stream()
                        .map(s -> {
                            int indexOfSharp = s.indexOf("#");
                            return s.substring(0, indexOfSharp == -1 ? s.length() : indexOfSharp);
                        }).collect(Collectors.toList());
            } else {//url正则式为空则抽取本域名下的所有连接,并使用黑名单对链接进行过滤
                links = page.getHtml().links()
                        .regex("https?://" + info.getDomain().replace(".", "\\.") + "/.*")
                        .all().stream().map(s -> {
                            int indexOfSharp = s.indexOf("#");
                            return s.substring(0, indexOfSharp == -1 ? s.length() : indexOfSharp);
                        })
                        .filter(s -> {
                            for (String ignoredPostfix : ignoredUrls) {
                                if (s.toLowerCase().endsWith(ignoredPostfix)) {
                                    return false;
                                }
                            }
                            return true;
                        }).collect(Collectors.toList());
            }
            //如果页面包含iframe则也进行抽取
            for (Element iframe : page.getHtml().getDocument().getElementsByTag("iframe")) {
                final String src = iframe.attr("src");
                //iframe抽取规则遵循设定的url正则
                if (StringUtils.isNotBlank(info.getUrlReg()) && src.matches(info.getUrlReg())) {
                    links.add(src);
                }
                //如无url正则,则遵循同源策略
                else if (StringUtils.isBlank(info.getUrlReg()) && UrlUtils.getDomain(src).equals(info.getDomain())) {
                    links.add(src);
                }
            }
            if (links != null && links.size() > 0) {
                page.addTargetRequests(links);
                //仅在debug模式下向任务管理系统存储链接信息
                if (staticValue.isCommonsSpiderDebug()) {
                    List<String> urls;
                    if ((urls = ((List<String>) task.getExtraInfoByKey(LINK_KEY))) != null) {
                        urls.addAll(links);
                    } else {
                        task.addExtraInfo(LINK_KEY, links);
                    }
                }
            }
        }
        //去掉startUrl页面
        if (startPage) {
            page.setSkip(true);
        }
        page.putField("url", page.getUrl().get());
        page.putField("domain", info.getDomain());
        page.putField("spiderInfoId", info.getId());
        page.putField("gatherTime", new Date());
        page.putField("spiderInfo", info);
        page.putField("spiderUUID", task.getTaskId());
        if (info.isSaveCapture()) {
            page.putField("rawHTML", page.getHtml().get());
        }
        //转换静态字段
        if (info.getStaticFields() != null && info.getStaticFields().size() > 0) {
            Map<String, String> staticFieldList = Maps.newHashMap();
            for (SpiderInfo.StaticField staticField : info.getStaticFields()) {
                staticFieldList.put(staticField.getName(), staticField.getValue());
            }
            page.putField("staticField", staticFieldList);
        }
        ///////////////////////////////////////////////////////
        String content;
        if (!StringUtils.isBlank(info.getContentXPath())) {//如果有正文的XPath的话优先使用XPath
            StringBuilder buffer = new StringBuilder();
            page.getHtml().xpath(info.getContentXPath()).all().forEach(buffer::append);
            content = buffer.toString();
        } else if (!StringUtils.isBlank(info.getContentReg())) {//没有正文XPath
            StringBuilder buffer = new StringBuilder();
            page.getHtml().regex(info.getContentReg()).all().forEach(buffer::append);
            content = buffer.toString();
        } else {//如果没有正文的相关规则则使用智能提取
            Document clone = page.getHtml().getDocument().clone();
            clone.getElementsByTag("p").append("***");
            clone.getElementsByTag("br").append("***");
            clone.getElementsByTag("script").remove();
            //移除不可见元素
            clone.getElementsByAttributeValueContaining("style", "display:none").remove();
            content = new Html(clone).smartContent().get();
        }
        content = content.replaceAll("<script([\\s\\S]*?)</script>", "");
        content = content.replaceAll("<style([\\s\\S]*?)</style>", "");
        content = content.replace("</p>", "***");
        content = content.replaceAll("<([\\s\\S]*?)>", "");

        content = content.replace("***", "<br/>");
        content = content.replace("\n", "<br/>");
        content = content.replaceAll("(\\<br/\\>\\s*){2,}", "<br/> ");
        content = content.replaceAll("(&nbsp;\\s*)+", " ");
        page.putField("content", content);
        if (info.isNeedContent() && StringUtils.isBlank(content)) {//if the content is blank ,skip it!
            page.setSkip(true);
            return;
        }
        //抽取标题
        String title = null;
        if (!StringUtils.isBlank(info.getTitleXPath())) {//提取网页标题
            title = page.getHtml().xpath(info.getTitleXPath()).get();
        } else if (!StringUtils.isBlank(info.getTitleReg())) {
            title = page.getHtml().regex(info.getTitleReg()).get();
        } else {//如果不写默认是title
            title = page.getHtml().getDocument().title();
        }
        page.putField("title", title);
        if (info.isNeedTitle() && StringUtils.isBlank(title)) {//if the title is blank ,skip it!
            page.setSkip(true);
            return;
        }

        //抽取动态字段
        Map<String, Object> dynamicFields = Maps.newHashMap();
        for (SpiderInfo.FieldConfig conf : info.getDynamicFields()) {
            String fieldName = conf.getName();
            String fieldData = null;
            if (!StringUtils.isBlank(conf.getXpath())) {//提取
                fieldData = page.getHtml().xpath(conf.getXpath()).get();
            } else if (!StringUtils.isBlank(conf.getRegex())) {
                fieldData = page.getHtml().regex(conf.getRegex()).get();
            }
            dynamicFields.put(fieldName, fieldData);
            if (conf.isNeed() && StringUtils.isBlank(fieldData)) {//if the field data is blank ,skip it!
                page.setSkip(true);
                return;
            }
        }
        page.putField(DYNAMIC_FIELD, dynamicFields);

        //抽取分类
        String category = null;
        if (!StringUtils.isBlank(info.getCategoryXPath())) {//提取网页分类
            category = page.getHtml().xpath(info.getCategoryXPath()).get();
        } else if (!StringUtils.isBlank(info.getCategoryReg())) {
            category = page.getHtml().regex(info.getCategoryReg()).get();
        }
        if (StringUtils.isNotBlank(category)) {
            page.putField("category", category);
        }

        //抽取发布时间
        String publishTime = null;
        if (!StringUtils.isBlank(info.getPublishTimeXPath())) {//文章发布时间规则
            publishTime = page.getHtml().xpath(info.getPublishTimeXPath()).get();
        } else if (!StringUtils.isBlank(info.getPublishTimeReg())) {
            publishTime = page.getHtml().regex(info.getPublishTimeReg()).get();
        }
        Date publishDate = null;
        SimpleDateFormat simpleDateFormat = null;
        //获取SimpleDateFormat时间匹配模板,首先检测爬虫模板指定的,如果为空则自动探测
        if (StringUtils.isNotBlank(info.getPublishTimeFormat())) {
            //使用爬虫模板指定的时间匹配模板
            if (StringUtils.isNotBlank(info.getLang())) {
                simpleDateFormat = new SimpleDateFormat(info.getPublishTimeFormat(), new Locale(info.getLang(), info.getCountry()));
            } else {
                simpleDateFormat = new SimpleDateFormat(info.getPublishTimeFormat());
            }
        } else if (StringUtils.isBlank(publishTime) && info.isAutoDetectPublishDate()) {
            //如果没有使用爬虫模板抽取到文章发布时间,或者选择了自动抽时间,则进行自动发布时间探测
            for (Pair<String, SimpleDateFormat> formatEntry : datePattern) {
                publishTime = page.getHtml().regex(formatEntry.getKey(), 0).get();
                //如果探测到了时间就退出探测
                if (StringUtils.isNotBlank(publishTime)) {
                    simpleDateFormat = formatEntry.getValue();
                    break;
                }
            }
        }
        //解析发布时间成date类型
        if (simpleDateFormat != null && StringUtils.isNotBlank(publishTime)) {
            try {
                publishDate = simpleDateFormat.parse(publishTime);
                //如果时间没有包含年份,则默认使用当前年
                if (!simpleDateFormat.toPattern().contains("yyyy")) {
                    Calendar calendar = Calendar.getInstance();
                    calendar.setTime(publishDate);
                    calendar.set(Calendar.YEAR, Calendar.getInstance().get(Calendar.YEAR));
                    publishDate = calendar.getTime();
                }
                page.putField("publishTime", publishDate);
            } catch (ParseException e) {
                LOG.debug("解析文章发布时间出错,source:" + publishTime + ",format:" + simpleDateFormat.toPattern());
                task.setDescription("解析文章发布时间出错,url:%s source:%s ,format:%s", page.getUrl().toString(), publishTime, simpleDateFormat.toPattern());
                if (info.isNeedPublishTime()) {//if the publishTime is blank ,skip it!
                    page.setSkip(true);
                    return;
                }
            }
        } else if (info.isNeedPublishTime()) {//if the publishTime is blank ,skip it!
            page.setSkip(true);
            return;
        }
        ///////////////////////////////////////////////////////
        if (info.isDoNLP()) {//判断本网站是否需要进行自然语言处理
            //进行nlp处理之前先去除标签
            String contentWithoutHtml = content.replaceAll("<br/>", "");
            try {
                //抽取关键词,10个词
                page.putField("keywords", keywordsExtractor.extractKeywords(contentWithoutHtml));
                //抽取摘要,5句话
                page.putField("summary", summaryExtractor.extractSummary(contentWithoutHtml));
                //抽取命名实体
                page.putField("namedEntity", namedEntitiesExtractor.extractNamedEntity(contentWithoutHtml));
            } catch (Exception e) {
                e.printStackTrace();
                LOG.error("对网页进行NLP处理失败,{}", e.getLocalizedMessage());
                task.setDescription("对网页进行NLP处理失败,%s", e.getLocalizedMessage());
            }
        }
        //本页面处理时长
        page.putField("processTime", System.currentTimeMillis() - start);
    };
    
    
    private List<Pipeline> pipelineList;
    private CommonWebpagePipeline commonWebpagePipeline;
    private ContentLengthLimitHttpClientDownloader contentLengthLimitHttpClientDownloader;
    private CommonWebpageDAO commonWebpageDAO;
    private SpiderInfoDAO spiderInfoDAO;

    @Autowired
    public CommonSpider(TaskManager taskManager, StaticValue staticValue) throws InterruptedException, BindException {
        this.taskManager = taskManager;
        this.staticValue = staticValue;
        Timer timer = new Timer();
        timer.schedule(new TimerTask() {
            @Override
            public void run() {
                deleteAll();
                LOG.debug("定时删除全部完成的普通网页抓取任务");
            }
        }, staticValue.getTaskDeleteDelay() * 3600000, staticValue.getTaskDeletePeriod() * 3600000);
        LOG.debug("定时删除普通网页抓取任务记录线程已启动,延时:{}小时,每{}小时删除一次", staticValue.getTaskDeleteDelay(), staticValue.getTaskDeletePeriod());
    }

    /**
     * 停止爬虫
     *
     * @param uuid
     */
    public void stop(String uuid) {
        Preconditions.checkArgument(spiderMap.containsKey(uuid), "找不到UUID为%s的爬虫,请检查参数", uuid);
        spiderMap.get(uuid).close();
        spiderMap.get(uuid).stop();
        taskManager.getTaskById(uuid).setState(State.STOP);
    }

    /**
     * 删除爬虫
     *
     * @param uuid
     */
    public void delete(String uuid) {
        Preconditions.checkArgument(spiderMap.containsKey(uuid) || taskManager.getTaskById(uuid) != null, "找不到UUID为%s的爬虫,请检查参数", uuid);
        Preconditions.checkArgument(taskManager.getTaskById(uuid).getState() == State.STOP, "爬虫" + uuid + "尚未停止,不能删除任务");
        deleteTaskById(uuid);
        spiderMap.remove(uuid);
    }

    /**
     * 删除全部爬虫
     */
    public void deleteAll() {
        List<String> spiderUUID2BeRemoved = spiderMap.entrySet().stream().filter(
                spiderEntry -> spiderEntry.getValue().getStatus() == Spider.Status.Stopped
        ).map(Map.Entry::getKey).collect(Collectors.toList());
        for (String uuid : spiderUUID2BeRemoved) {
            try {
                deleteTaskById(uuid);
                spiderMap.remove(uuid);
            } catch (Exception e) {
                LOG.error("删除任务ID:{}出错,{}", uuid, e.getLocalizedMessage());
            }
        }
        taskManager.deleteTasksByState(State.STOP);
    }

    /**
     * 根据ID获取爬虫对象
     *
     * @param uuid
     * @return
     */
    public MySpider getSpiderById(String uuid) {
        Preconditions.checkArgument(spiderMap.containsKey(uuid), "找不到UUID为%s的爬虫,请检查参数", uuid);
        return spiderMap.get(uuid);
    }

    /**
     * 启动爬虫
     *
     * @param info 爬虫模板信息
     * @return
     * @throws JMException
     */
    public String start(SpiderInfo info) throws JMException {
        boolean running = taskManager.findTaskBy(task -> {
            Object spiderinfoObj = task.getExtraInfoByKey(SPIDER_INFO);
            if (spiderinfoObj != null && spiderinfoObj instanceof SpiderInfo) {
                SpiderInfo spiderInfo = (SpiderInfo) spiderinfoObj;
                return task.getState() == State.RUNNING && spiderInfo.getId().equals(info.getId());
            }
            return false;
        });
        Preconditions.checkArgument(!running, "已经提交了这个任务,模板编号%s,请勿重复提交", info.getId());
        final String uuid = UUID.randomUUID().toString();
        Task task = taskManager.initTask(uuid, info.getDomain(), info.getCallbackURL(), "spiderInfoId=" + info.getId() + "&spiderUUID=" + uuid);
        task.addExtraInfo(SPIDER_INFO, info);
        QueueScheduler scheduler = new QueueScheduler() {
            @Override
            public void pushWhenNoDuplicate(Request request, us.codecraft.webmagic.Task task) {
                int left = getLeftRequestsCount(task);
                if (left <= staticValue.getLimitOfCommonWebpageDownloadQueue()) {
                    super.pushWhenNoDuplicate(request, task);
                }
            }
        };
        if (staticValue.isNeedEs()) {
            scheduler.setDuplicateRemover(commonWebpagePipeline);  //调用ES的存储操作函数commonWebpagePipeline
        }
        MySpider spider = (MySpider) makeSpider(info, task)
                .setScheduler(scheduler);
        //添加其他的数据管道
        if (pipelineList != null && pipelineList.size() > 0) {
            pipelineList.forEach(spider::addPipeline);
        }
        info.getStartURL().forEach(s -> scheduler.pushWhenNoDuplicate(new Request(s), spider));
        //慎用爬虫监控,可能导致内存泄露
//        spiderMonitor.register(spider);
        spiderMap.put(uuid, spider);
        spider.start();  //
        taskManager.getTaskById(uuid).setState(State.RUNNING);
        return uuid;
    }

    /**
     * 测试爬虫模板
     *
     * @param info
     * @return
     */
    public List<Webpage> testSpiderInfo(SpiderInfo info) throws JMException {
        final ResultItemsCollectorPipeline resultItemsCollectorPipeline = new ResultItemsCollectorPipeline();
        final String uuid = UUID.randomUUID().toString();
        Task task = taskManager.initTask(uuid, info.getDomain(), info.getCallbackURL(), "spiderInfoId=" + info.getId() + "&spiderUUID=" + uuid);
        task.addExtraInfo("spiderInfo", info);
        QueueScheduler queueScheduler = new QueueScheduler();
        MySpider spider = (MySpider) makeSpider(info, task)
                .addPipeline(resultItemsCollectorPipeline)
                .setScheduler(queueScheduler);
        spider.startUrls(info.getStartURL());
        //慎用爬虫监控,可能导致内存泄露
//        spiderMonitor.register(spider);
        spiderMap.put(uuid, spider);
        taskManager.getTaskById(uuid).setState(State.RUNNING);
        spider.run();
        
        //将所抽取出来的爬虫数据由key-value形式转换为实体对象
        List<Webpage> webpageList = Lists.newLinkedList();
        resultItemsCollectorPipeline.getCollected().forEach(resultItems -> webpageList.add(CommonWebpagePipeline.convertResultItems2Webpage(resultItems)));
        return webpageList;
    }

    /**
     * 列出所有爬虫的运行时信息
     *
     * @return
     */
    public Map<String, Map<Object, Object>> listAllSpiders(boolean containsExtraInfo) {
        Map<String, Map<Object, Object>> result = new HashMap<>();
        spiderMap.entrySet().forEach(spiderEntry -> {
            MySpider spider = spiderEntry.getValue();
            result.put(spiderEntry.getKey(), makeSpiderRuntimeInfo(spider, containsExtraInfo));
        });
        return result;
    }

    /**
     * 获取爬虫运行时信息
     *
     * @param uuid 爬虫uuid 任务id
     * @return
     */
    public Map<Object, Object> getSpiderRuntimeInfo(String uuid, boolean containsExtraInfo) {
        return makeSpiderRuntimeInfo(getSpiderById(uuid), containsExtraInfo);
    }

    /**
     * 获取爬虫运行时信息
     *
     * @param spider
     * @return
     */
    private Map<Object, Object> makeSpiderRuntimeInfo(MySpider spider, boolean containsExtraInfo) {
        Map<Object, Object> infoMap = new HashMap<>();
        infoMap.put("PageCount", spider.getPageCount());
        infoMap.put("StartTime", spider.getStartTime());
        infoMap.put("ThreadAlive", spider.getThreadAlive());
        infoMap.put("Status", spider.getStatus());
        infoMap.put("SpiderInfo", spider.getSpiderInfo());
        if (containsExtraInfo) {
            infoMap.put("Links", getTaskById(spider.getUUID(), true).getExtraInfoByKey(LINK_KEY));
        }
        return infoMap;
    }

    /**
     * 获取忽略url黑名单
     *
     * @return
     */
    public List<String> getIgnoredUrls() {
        return ignoredUrls;
    }

    /**
     * 添加忽略url黑名单
     *
     * @param postfix
     */
    public void addIgnoredUrl(String postfix) {
        Preconditions.checkArgument(!ignoredUrls.contains(postfix), "已包含这个url后缀请勿重复添加");
        ignoredUrls.add(postfix);
    }

    /**
     * 根据网站domain删除数据
     *
     * @param domain 网站域名
     * @return 是否全部数据删除成功
     */
    public String deleteByDomain(String domain) {
        final String uuid = UUID.randomUUID().toString();
        Task task = taskManager.initTask(uuid, "DELETE BY DOMAIN:" + domain, Lists.newArrayList(), null);
        Thread thread = new Thread(() -> {
            task.setState(State.RUNNING);
            try {
                commonWebpageDAO.deleteByDomain(domain, task);
            } catch (Exception e) {
                task.setDescription("删除数据时发生异常{}", e.toString() + e.getLocalizedMessage());
            } finally {
                task.setState(State.STOP);
            }
        }, "delete-webpage-thread-" + domain);
        thread.start();
        return uuid;
    }

    /**
     * 根据spiderinfoID更新数据
     *
     * @param spiderInfoIdUpdateBy 待更新网站模板编号
     * @param callbackUrls         回调地址
     * @param spiderInfo           新的网页抽取模板
     * @return 是否全部数据删除成功
     */
    public String updateBySpiderinfoID(String spiderInfoIdUpdateBy, SpiderInfo spiderInfo, List<String> callbackUrls) {
        Preconditions.checkArgument(StringUtils.isNotBlank(spiderInfo.getId()), "新模板必须含有ID字段");
        SpiderInfo inDatabase = spiderInfoDAO.getById(spiderInfo.getId());
        Preconditions.checkArgument(inDatabase != null, "新模板必须在模板库中注册");
        final String uuid = UUID.randomUUID().toString();
        Task task = taskManager.initTask(uuid, "Update By SpiderinfoID:" + spiderInfoIdUpdateBy, callbackUrls, "spiderInfoIdUpdateBy" + spiderInfoIdUpdateBy);
        Thread thread = new Thread(() -> {
            task.setState(State.RUNNING);
            try {
                Pair<String, List<Webpage>> pair = commonWebpageDAO.startScroll(QueryBuilders.matchQuery("spiderInfoId", spiderInfoIdUpdateBy).operator(Operator.AND), 50);
                int scrollPage = 0;//滚动到第几页了
                List<Webpage> webpageList = pair.getRight();
                while (webpageList.size() > 0) {
                    List<Webpage> newWebpageList = Lists.newArrayList();
                    for (Webpage webpage : webpageList) {
                        try {
                            //构建page对象
                            Page page = new Page();
                            Request request = new Request(webpage.getUrl());
                            page.setRequest(request);
                            page.setRawText(webpage.getRawHTML());
                            page.setUrl(new PlainText(webpage.getUrl()));
                            //应用新模板抽取数据
                            spiderInfoPageConsumer.accept(page, spiderInfo, task);
                            //更新网页数据
                            newWebpageList.add(CommonWebpagePipeline.convertResultItems2Webpage(page.getResultItems()));
                        } catch (Exception e) {
                            LOG.error("应用模板时发生异常,webpageID:{},error:{}", webpage.getId(), e.getLocalizedMessage());
                        } finally {
                            task.increaseCount();
                        }
                    }
                    //更新库中数据
                    boolean hasFail = commonWebpageDAO.update(newWebpageList);
                    task.setDescription("已经更新%s页数据,错误:%s", ++scrollPage, hasFail);
                    webpageList = commonWebpageDAO.scrollAllWebpage(pair.getLeft());
                }
            } catch (Exception e) {
                task.setDescription("根据spiderinfoID更新数据时发生异常%s", e.toString() + e.getLocalizedMessage());
            } finally {
                task.setState(State.STOP);
                taskManager.stopTask(task);
            }
        }, "webpage-update-thread-" + spiderInfoIdUpdateBy);
        thread.start();
        return uuid;
    }

    /**
     * 生成爬虫
     *
     * @param info 抓取模板
     * @param task 任务实体
     * @return
     */
    private MySpider makeSpider(SpiderInfo info, Task task) {
        return ((MySpider) new MySpider(new MyPageProcessor(info, task), info)
                .thread(info.getThread())
                .setDownloader(contentLengthLimitHttpClientDownloader)
                .setUUID(task.getTaskId()));
    }

    public NLPExtractor getKeywordsExtractor() {
        return keywordsExtractor;
    }

    public void setKeywordsExtractor(NLPExtractor keywordsExtractor) {
        this.keywordsExtractor = keywordsExtractor;
    }

    public CommonWebpagePipeline getCommonWebpagePipeline() {
        return commonWebpagePipeline;
    }

    public CommonSpider setCommonWebpagePipeline(CommonWebpagePipeline commonWebpagePipeline) {
        this.commonWebpagePipeline = commonWebpagePipeline;
        return this;
    }

    public NLPExtractor getSummaryExtractor() {
        return summaryExtractor;
    }

    public void setSummaryExtractor(NLPExtractor summaryExtractor) {
        this.summaryExtractor = summaryExtractor;
    }

    public NLPExtractor getNamedEntitiesExtractor() {
        return namedEntitiesExtractor;
    }

    public CommonSpider setNamedEntitiesExtractor(NLPExtractor namedEntitiesExtractor) {
        this.namedEntitiesExtractor = namedEntitiesExtractor;
        return this;
    }

    public ContentLengthLimitHttpClientDownloader getContentLengthLimitHttpClientDownloader() {
        return contentLengthLimitHttpClientDownloader;
    }

    public CommonSpider setContentLengthLimitHttpClientDownloader(ContentLengthLimitHttpClientDownloader contentLengthLimitHttpClientDownloader) {
        this.contentLengthLimitHttpClientDownloader = contentLengthLimitHttpClientDownloader;
        return this;
    }

    public CommonWebpageDAO getCommonWebpageDAO() {
        return commonWebpageDAO;
    }

    public CommonSpider setCommonWebpageDAO(CommonWebpageDAO commonWebpageDAO) {
        this.commonWebpageDAO = commonWebpageDAO;
        return this;
    }

    public SpiderInfoDAO getSpiderInfoDAO() {
        return spiderInfoDAO;
    }

    public CommonSpider setSpiderInfoDAO(SpiderInfoDAO spiderInfoDAO) {
        this.spiderInfoDAO = spiderInfoDAO;
        return this;
    }

    public List<Pipeline> getPipelineList() {
        return pipelineList;
    }

    public CommonSpider setPipelineList(List<Pipeline> pipelineList) {
        this.pipelineList = pipelineList;
        return this;
    }

    /**
     * 在原有的webmagic基础上添加了一些其他功能
     */
    private class MySpider extends Spider {
        private final SpiderInfo SPIDER_INFO;
        private Logger LOG = LogManager.getLogger(MySpider.class);

        MySpider(PageProcessor pageProcessor, SpiderInfo spiderInfo) {
            super(pageProcessor);
            this.SPIDER_INFO = spiderInfo;
        }

        @Override
        protected void onSuccess(Request request) {
            super.onSuccess(request);
            Task task = taskManager.getTaskById(this.getUUID());
            boolean reachMax = false, exceedRatio = false;
            if (
                    (
                            //已抓取数量大于最大抓取页数,退出
                            (reachMax = (SPIDER_INFO.getMaxPageGather() > 0 && task.getCount() >= SPIDER_INFO.getMaxPageGather()))
                                    ||
                                    //如果抓取页面超过最大抓取数量ratio倍的时候,仍未达到最大抓取数量,爬虫也退出
                                    (exceedRatio = (this.getPageCount() > SPIDER_INFO.getMaxPageGather() * staticValue.getCommonsWebpageCrawlRatio()))
                    )
                            && this.getStatus() == Status.Running) {
                LOG.info("爬虫ID{}已处理{}个页面,有效页面{}个,最大抓取页数{},reachMax={},exceedRatio={},退出.", this.getUUID(), this.getPageCount(), task.getCount(), SPIDER_INFO.getMaxPageGather(), reachMax, exceedRatio);
                task.setDescription("爬虫ID%s已处理%s个页面,有效页面%s个,达到最大抓取页数%s,reachMax=%s,exceedRatio=%s,退出.", this.getUUID(), this.getPageCount(), task.getCount(), SPIDER_INFO.getMaxPageGather(), reachMax, exceedRatio);
                this.stop();
            }
        }

        @Override
        protected void onError(Request request) {
            super.onError(request);
            Task task = taskManager.getTaskById(this.getUUID());
            task.setDescription("处理网页%s时发生错误,%s", request.getUrl(), request.getExtras());
        }

        @Override
        public void close() {
            super.close();
            Task task = taskManager.getTaskById(this.getUUID());
            if (task != null) {
                //清除抓取列表缓存
                commonWebpagePipeline.deleteUrls(task.getTaskId());
                taskManager.stopTask(task);
            }
            pageProcessor = null;
        }

        SpiderInfo getSpiderInfo() {
            return SPIDER_INFO;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;

            if (o == null || getClass() != o.getClass()) return false;

            MySpider mySpider = (MySpider) o;

            return new EqualsBuilder()
                    .append(this.getUUID(), mySpider.getUUID())
                    .isEquals();
        }

        @Override
        public int hashCode() {
            return new HashCodeBuilder(17, 37)
                    .append(this.getUUID())
                    .toHashCode();
        }
    }

    private class MyPageProcessor implements PageProcessor {

        private Site site;
        private SpiderInfo info;
        private Task task;

        public MyPageProcessor(SpiderInfo info, Task task) {
            this.site = Site.me().setDomain(info.getDomain()).setTimeOut(info.getTimeout())
                    .setRetryTimes(info.getRetry()).setSleepTime(info.getSleep())
                    .setCharset(StringUtils.isBlank(info.getCharset()) ? null : info.getCharset())
                    .setUserAgent(info.getUserAgent());
            this.info = info;
            this.task = task;
        }

        @Override
        public void process(Page page) {
            spiderInfoPageConsumer.accept(page, info, task);
            if (!page.getResultItems().isSkip()) {//网页正常时再增加数量
                task.increaseCount();
            }
        }

        @Override
        public Site getSite() {
            return site;
        }
    }
}
