<reponame>ecadagiani/webFinder
{
    "debug": false,
    "showPluginTimeoutError": true,
    "throwError": false,
    "loop": true,
    "start": "",
    "searchSelectors": [],
    "browserLanguage": "en-US",
    "browserOptions": {
        "ignoreHTTPSErrors": true,
        "headless": true
    },
    "pluginTimeout": 3000,
    "timeBetweenTwoFetch": 1000,
    "waitForPageLoadTimeout": 30000,
    "timeoutForBodyAppear": 5000,
    "waitForBodyAppear": true,
    "maxErrorFetchPage": 3,
    "maxErrorGetNewLink": 3,
    "maxUrlLength": 800,
    "crawlInvisibleLink": false,
    "domainScoreFunction": "(domain, nbFetch) => nbFetch < 50 ? 0 : Math.floor( (nbFetch-50) * -5 )",
    "interestMinimumScoreToContinue": 0,
    "interestMinimumScoreToFetchDb": 0,
    "interestRandRange": {
        "min": 0,
        "max": 2
    },
    "mergeAuthorizedLinksExtensions": true,
    "authorizedLinksExtensions": [
        "html",
        "html5",
        "htm",
        "php",
        "php3"
    ],
    "mergeAuthorizedURIScheme": true,
    "authorizedURIScheme": [
        "http",
        "https",
        "shttp"
    ],
    "interestLanguage": [],
    "interestLanguageImpact": 20,
    "uninterestLanguageImpact": -10,
    "domainWhitelist": [
        "libraryofbabel.info",
        "theinfinitelibrary.com"
    ],
    "mergeDomainWhitelist": true,
    "interestTag": [],
    "interestTagDefaultImpact": 1,
    "mergeInterestTag": true,
    "uninterestingTag": [],
    "uninterestingTagDefaultImpact": -1,
    "mergeUninterestTag": true,
    "interestTagUrl": [],
    "interestTagUrlDefaultImpact": 2,
    "mergeInterestTagUrl": true,
    "uninterestingTagUrl": [
        {
            "tag": "admin",
            "impact": -1
        },
        "cart",
        "checkout",
        "favorite",
        "password",
        "register",
        "registration",
        "sendfriend",
        "wishlist,",
        "signin",
        "login",
        "cgi-bin",
        "includes",
        "var,",
        "filter",
        "limit",
        "order",
        "sort,",
        "sessionid",
        "session_id",
        "SID",
        "PHPSESSID",
        "add"
    ],
    "uninterestingTagUrlDefaultImpact": -2,
    "mergeUninterestTagUrl": true,
    "searchTags": [
        "foo",
        "bar",
        "crawler"
    ],
    "maxCombinationSearchTags": 2,
    "offsetMaxSearchEngine": 100,
    "searchEngineLanguage": "uk-en",


    "mongo": {
        "host": "",
        "port": "",
        "database": "",
        "username": "",
        "password": "",
        "maxConnectionTry": 3,
        "timeBetweenEachConnectionTry": 4000
    },
    "mongoSampleSize": 1000,


    "loopMaxTimeout": 240000,
    "managerServerPort": 3000,
    "nbCrawler": 1,
    "crawlerProcessExitWait": 2000
}
