package de.fhg.iais.cortex.jobscheduler.crawler;

import java.io.File;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.FileUtils;
import org.apache.http.Header;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Matchers;
import org.mockito.Mockito;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

import de.fhg.iais.commons.exception.StorageException;
import de.fhg.iais.cortex.jobscheduler.crawler.handler.IParseDataHandler;
import de.fhg.iais.cortex.jobscheduler.crawler.hibernate.CrawledWebPage;
import de.fhg.iais.cortex.jobscheduler.crawler.hibernate.WebsiteCrawlerStats;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

public class WebsiteCrawlerTest {

    private WebsiteCrawler crawler;
    private CrawlController controllerMock;
    private CrawlConfig crawlConfigMock;
    private WebsiteCrawlerConfiguration ascCrawlConfigMock;
    private IParseDataHandler reqHandler;

    private WebsiteCrawlerStatistics statisticsMock;
    private WebsiteCrawlerPages pagesMock;

    private WebsiteCrawlerStats stats;

    private Map<String, String> crawlerAttribs;
    private File storagePath;
    private int deleteAfterCount;

    private final String CRAWLER_NAME = "CRAWLER_NAME";

    @Before
    public void setup() throws URISyntaxException {
        this.reqHandler = Mockito.mock(IParseDataHandler.class);
        this.crawlerAttribs = Maps.newHashMap();

        this.statisticsMock = Mockito.mock(WebsiteCrawlerStatistics.class);
        this.stats = Mockito.mock(WebsiteCrawlerStats.class);
        Mockito.when(this.statisticsMock.getStats(this.CRAWLER_NAME)).thenReturn(this.stats);

        this.pagesMock = Mockito.mock(WebsiteCrawlerPages.class);

        this.deleteAfterCount = 3;

        this.crawler = new WebsiteCrawler(this.reqHandler, this.statisticsMock, this.pagesMock, this.deleteAfterCount);
        this.controllerMock = Mockito.mock(CrawlController.class);
        this.crawlConfigMock = Mockito.mock(CrawlConfig.class);
        this.ascCrawlConfigMock = Mockito.mock(WebsiteCrawlerConfiguration.class);
        Mockito.when(this.ascCrawlConfigMock.getStoragePath()).thenReturn(this.storagePath);
        Mockito.when(this.ascCrawlConfigMock.getCrawlerAttribs()).thenReturn(this.crawlerAttribs);
        Mockito.when(this.ascCrawlConfigMock.getCrawlerName()).thenReturn(this.CRAWLER_NAME);
        Mockito.when(this.controllerMock.getCustomData()).thenReturn(this.ascCrawlConfigMock);
        Mockito.when(this.controllerMock.getConfig()).thenReturn(this.crawlConfigMock);
        Mockito.when(this.controllerMock.isShuttingDown()).thenReturn(false);
    }

    @Test
    public void testInit() {
        this.crawler = new WebsiteCrawler(this.reqHandler, this.statisticsMock, this.pagesMock, this.deleteAfterCount);

        this.crawler.init(1, this.controllerMock);

        Mockito.verify(this.controllerMock).getCustomData();
        Mockito.verify(this.statisticsMock, Mockito.times(1)).getStats(this.CRAWLER_NAME);
        Mockito.verify(this.statisticsMock, Mockito.times(0)).createStats(this.CRAWLER_NAME);
        Mockito.verify(this.stats).setTimeStarted(Matchers.anyLong());
    }

    @Test
    public void testInitExceptions() {

        this.statisticsMock = Mockito.mock(WebsiteCrawlerStatistics.class);
        Mockito.when(this.statisticsMock.getStats(this.CRAWLER_NAME)).thenThrow(new StorageException(""));
        Mockito.when(this.statisticsMock.createStats(this.CRAWLER_NAME)).thenReturn(this.stats);

        this.crawler = new WebsiteCrawler(this.reqHandler, this.statisticsMock, this.pagesMock, this.deleteAfterCount);

        this.crawler.init(1, this.controllerMock);

        Mockito.verify(this.controllerMock).getCustomData();
        Mockito.verify(this.statisticsMock, Mockito.times(1)).getStats(this.CRAWLER_NAME);
        Mockito.verify(this.statisticsMock, Mockito.times(1)).createStats(this.CRAWLER_NAME);
        Mockito.verify(this.stats).setTimeStarted(Matchers.anyLong());

    }

    @Test
    public void testShouldVisit() {
        this.crawler.init(1, this.controllerMock);

        Mockito.when(this.statisticsMock.getStats(this.CRAWLER_NAME)).thenReturn(this.stats);
        final WebURL validUrl = new WebURL();
        validUrl.setURL("http://somestringWhich.de/getParsed.pdf");

        final Page page = null;
        final WebURL invalidUrl = new WebURL();
        invalidUrl.setURL("http://somestringWhich.de/getParsed_old.pdf");

        final List<Pattern> positivePattern = Lists.newArrayList(Pattern.compile(".*?\\.pdf"));
        final List<Pattern> negativePattern = Lists.newArrayList(Pattern.compile(".*?_old\\.pdf"));

        WebsiteCrawlerStats stats = new WebsiteCrawlerStats();
        Mockito.when(this.statisticsMock.getStats(this.CRAWLER_NAME)).thenReturn(stats);
        Mockito.when(this.ascCrawlConfigMock.getPositivePattern()).thenReturn(positivePattern);
        Mockito.when(this.ascCrawlConfigMock.getNegativePattern()).thenReturn(negativePattern);

        Assert.assertTrue(this.crawler.shouldVisit(page, validUrl));
        Assert.assertFalse(this.crawler.shouldVisit(page, invalidUrl));

        Mockito.verify(this.stats, Mockito.times(2)).incTotalLinks(1);
    }

    @Test
    public void testVisit() {
        this.crawler.init(1, this.controllerMock);

        final String validUrl = "http://somestringWhich.de/getParsed.pdf";

        final WebURL validUrlAsWebURL = new WebURL();
        validUrlAsWebURL.setURL(validUrl);

        final Page page = new Page(validUrlAsWebURL);
        final HtmlParseData data = new HtmlParseData();
        final String html = "<html><head></head><body></body></html>";
        final int statusCode = 200;
        data.setHtml(html);
        page.setParseData(data);
        page.setStatusCode(statusCode);
        page.setContentData(data.getHtml().getBytes());
        page.setFetchResponseHeaders(new Header[0]);

        final String previousContentHash = "ABCDEFGHIZ";

        CrawledWebPage cPage = new CrawledWebPage();
        cPage.incDeleteCount();
        cPage.setContent(previousContentHash);
        Mockito.when(this.pagesMock.getPageByUrlAndCrawlerName(validUrl, this.CRAWLER_NAME)).thenReturn(cPage);

        this.crawler.visit(page);

        Assert.assertEquals(0, cPage.getDeleteCount());

        Mockito.verify(this.reqHandler, Mockito.times(1)).handle(page, this.crawlerAttribs);
        Mockito.verify(this.reqHandler, Mockito.times(1)).delete(previousContentHash);

        Mockito.verify(this.pagesMock, Mockito.times(1)).storePage(cPage);
        Mockito.verify(this.stats, Mockito.times(1)).incTotalTextSize(html.getBytes().length);
        Mockito.verify(this.stats, Mockito.times(1)).incProcessedPages();
        Mockito.verify(this.stats, Mockito.times(1)).incTotalIndexedDocuments();
        Mockito.verify(this.stats, Mockito.times(1)).incHttpCode(statusCode);

    }

    @Test
    public void testOnBiggerPageSize() {
        this.crawler.init(1, this.controllerMock);

        final String expectedUrl = "http://some.url/u/know";
        final long expectedLastModified = 0;
        final int expectedStatus = 0;
        final String expectedLastError = "Maximum size exceeded.";
        final String expectedHashedContent = "";
        final int expectedDeleteCount = 1;

        CrawledWebPage expectedPage = new CrawledWebPage();

        Mockito.when(this.pagesMock.getPageByUrlAndCrawlerName(expectedUrl, this.CRAWLER_NAME)).thenReturn(expectedPage);

        this.crawler.onPageBiggerThanMaxSize(expectedUrl, 0L);

        Assert.assertEquals(expectedPage.getCrawlerName(), this.CRAWLER_NAME);
        Assert.assertEquals(expectedPage.getContent(), expectedHashedContent);
        Assert.assertEquals(expectedPage.getLastError(), expectedLastError);
        Assert.assertTrue(expectedPage.getLastModified() > expectedLastModified);
        Assert.assertEquals(expectedPage.getName(), expectedUrl);
        Assert.assertEquals(expectedPage.getLastStatus(), expectedStatus);
        Assert.assertEquals(expectedPage.getDeleteCount(), expectedDeleteCount);

        Mockito.verify(this.stats).incErrorCount();
        Mockito.verify(this.pagesMock).getPageByUrlAndCrawlerName(expectedUrl, this.CRAWLER_NAME);

    }

    @Test
    public void testOnUnexpectedError() {
        this.crawler.init(1, this.controllerMock);

        final String expectedUrl = "http://some.url/u/know";
        final long expectedLastModified = 0;
        final int expectedStatus = HttpStatus.SC_GONE;
        final String expectedHashedContent = "ABFHDFGH";
        final int expectedDeleteCount = 1;

        CrawledWebPage expectedPage = new CrawledWebPage();
        expectedPage.setContent(expectedHashedContent);

        Mockito.when(this.pagesMock.getPageByUrlAndCrawlerName(expectedUrl, this.CRAWLER_NAME)).thenReturn(expectedPage);

        this.crawler.onUnexpectedStatusCode(expectedUrl, expectedStatus, "", "");

        Assert.assertEquals(expectedPage.getCrawlerName(), this.CRAWLER_NAME);
        Assert.assertEquals(expectedPage.getContent(), expectedHashedContent);
        Assert.assertTrue(!"".equals(expectedPage.getLastError()));
        Assert.assertTrue(expectedPage.getLastModified() > expectedLastModified);
        Assert.assertEquals(expectedPage.getName(), expectedUrl);
        Assert.assertEquals(expectedPage.getLastStatus(), expectedStatus);
        Assert.assertEquals(expectedPage.getDeleteCount(), expectedDeleteCount);

        Mockito.verify(this.stats).incErrorCount();
        Mockito.verify(this.pagesMock, Mockito.times(2)).getPageByUrlAndCrawlerName(expectedUrl, this.CRAWLER_NAME);
        Mockito.verify(this.reqHandler).delete(expectedHashedContent);
        Mockito.verify(this.pagesMock).deletePage(expectedUrl);
    }

    @After
    public void tearDown() {
        FileUtils.deleteQuietly(this.storagePath);
    }

}
