SpiderDetectorServiceImpl implements SpiderDetectorService { public boolean isSpider(String clientIP, String proxyIPs, String hostname, String agent) { if (null != agent) { synchronized(agents) { if (agents.isEmpty()) loadPatterns("agents", agents); } if(isUseCaseInsensitiveMatching()) { agent = StringUtils.lowerCase(agent); hostname = StringUtils.lowerCase(hostname); } for (Pattern candidate : agents) { if (null != candidate && candidate.matcher(agent).find()) { return true; } } } if (isUseProxies() && proxyIPs != null) { for (String xfip : proxyIPs.split(",")) { if (isSpider(xfip)) { return true; } } } if (isSpider(clientIP)) return true; if (null != hostname) { synchronized(domains) { if (domains.isEmpty()) loadPatterns("domains", domains); } for (Pattern candidate : domains) { if (null != candidate && candidate.matcher(hostname).find()) { return true; } } } return false; } @Autowired(required = true)  SpiderDetectorServiceImpl(ConfigurationService configurationService); IPTable getTable(); boolean isSpider(String clientIP, String proxyIPs, String hostname, String agent); Set<String> readPatterns(File patternFile); boolean isSpider(HttpServletRequest request); boolean isSpider(String ip); synchronized void loadSpiderIpAddresses();  }
@Test public void testCaseInsensitiveMatching() throws Exception { configurationService.setProperty("usage-statistics.bots.case-insensitive", true); spiderDetectorService = new SpiderDetectorServiceImpl(configurationService); DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnboT Is WaTching you"); assertTrue("'msnbot' didn't match pattern", spiderDetectorService.isSpider(req)); req.setAgent("FirefOx"); assertFalse("'Firefox' matched a pattern", spiderDetectorService.isSpider(req)); candidate = "192.168.2.1"; req.setAddress(candidate); assertTrue(candidate + " did not match IP patterns", spiderDetectorService.isSpider(req)); req.setAddress(NOT_A_BOT_ADDRESS); assertFalse(NOT_A_BOT_ADDRESS + " matched IP patterns", spiderDetectorService.isSpider(req)); candidate = "baiduspiDer-dSPace-test.crawl.baIDu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " did match DNS patterns", spiderDetectorService.isSpider(req)); candidate = "wIki.dsPace.oRg"; req.setRemoteHost(candidate); assertFalse(candidate + " matched DNS patterns", spiderDetectorService.isSpider(req)); }
@Test public void testIsSpiderHttpServletRequest() { DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnbot is watching you"); assertTrue("'msnbot' did not match any pattern", spiderDetectorService.isSpider(req)); req.setAgent("Firefox"); assertFalse("'Firefox' matched a pattern", spiderDetectorService.isSpider(req)); candidate = "192.168.2.1"; req.setAddress(candidate); assertTrue(candidate + " did not match IP patterns", spiderDetectorService.isSpider(req)); req.setAddress(NOT_A_BOT_ADDRESS); assertFalse(NOT_A_BOT_ADDRESS + " matched IP patterns", spiderDetectorService.isSpider(req)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " did not match DNS patterns", spiderDetectorService.isSpider(req)); candidate = "wiki.dspace.org"; req.setRemoteHost(candidate); assertFalse(candidate + " matched DNS patterns", spiderDetectorService.isSpider(req)); }
@Test public void testIsSpiderStringStringStringString() { String candidate; candidate = "192.168.2.1"; assertTrue(candidate + " did not match IP patterns", spiderDetectorService.isSpider(candidate, null, null, null)); candidate = NOT_A_BOT_ADDRESS; assertFalse(candidate + " matched IP patterns", spiderDetectorService.isSpider(candidate, null, null, null)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; assertTrue(candidate + " did not match DNS patterns", spiderDetectorService.isSpider(NOT_A_BOT_ADDRESS, null, candidate, null)); candidate = "wiki.dspace.org"; assertFalse(candidate + " matched DNS patterns", spiderDetectorService.isSpider(NOT_A_BOT_ADDRESS, null, candidate, null)); candidate = "msnbot is watching you"; assertTrue("'" + candidate + "' did not match agent patterns", spiderDetectorService.isSpider(NOT_A_BOT_ADDRESS, null, null, candidate)); candidate = "Firefox"; assertFalse("'" + candidate + "' matched agent patterns", spiderDetectorService.isSpider(NOT_A_BOT_ADDRESS, null, null, candidate)); }
@Test public void testIsSpiderString() { String candidate; candidate = "192.168.2.1"; assertTrue(candidate + " did not match IP patterns", spiderDetectorService.isSpider(candidate, null, null, null)); candidate = NOT_A_BOT_ADDRESS; assertFalse(candidate + " matched IP patterns", spiderDetectorService.isSpider(candidate, null, null, null)); }
@Test public void testCaseSensitiveMatching() throws Exception { DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnboT Is WaTching you"); assertFalse("'msnbot' matched pattern", spiderDetectorService.isSpider(req)); req.setAgent("FirefOx"); assertFalse("'Firefox' matched a pattern", spiderDetectorService.isSpider(req)); candidate = "192.168.2.1"; req.setAddress(candidate); assertTrue(candidate + " did not match IP patterns", spiderDetectorService.isSpider(req)); req.setAddress(NOT_A_BOT_ADDRESS); assertFalse(NOT_A_BOT_ADDRESS + " matched IP patterns", spiderDetectorService.isSpider(req)); candidate = "baiduspiDer-dSPace-test.crawl.baIDu.com"; req.setRemoteHost(candidate); assertFalse(candidate + " did match DNS patterns", spiderDetectorService.isSpider(req)); candidate = "wIki.dsPace.oRg"; req.setRemoteHost(candidate); assertFalse(candidate + " matched DNS patterns", spiderDetectorService.isSpider(req)); }
@Test public void testInsensitiveSensitiveDifference() { DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnbot is WaTching you"); assertTrue("'msnbot' didn't match pattern", spiderDetectorService.isSpider(req)); req.setAgent("MSNBOT Is WaTching you"); assertFalse("'msnbot' matched pattern", spiderDetectorService.isSpider(req)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " did not match DNS patterns", spiderDetectorService.isSpider(req)); candidate = "baiduspiDer-dSPace-test.crawl.baIDu.com"; req.setRemoteHost(candidate); assertFalse(candidate + " matched DNS patterns", spiderDetectorService.isSpider(req)); }
@Test public void testBothLowerAndUpperCaseGetMatched() { configurationService.setProperty("usage-statistics.bots.case-insensitive", true); spiderDetectorService = new SpiderDetectorServiceImpl(configurationService); DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnbot is WaTching you"); assertTrue("'msnbot' didn't match pattern", spiderDetectorService.isSpider(req)); req.setAgent("MSNBOT Is WaTching you"); assertTrue("'msnbot' didn't match pattern", spiderDetectorService.isSpider(req)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " did not match DNS patterns", spiderDetectorService.isSpider(req)); candidate = "baiduspiDer-dSPace-test.crawl.baIDu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " didn't match DNS patterns", spiderDetectorService.isSpider(req)); }
@Test public void testNonBooleanConfig() { configurationService.setProperty("usage-statistics.bots.case-insensitive", "RandomNonBooleanString"); spiderDetectorService = new SpiderDetectorServiceImpl(configurationService); DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnbot is WaTching you"); assertTrue("'msnbot' didn't match pattern", spiderDetectorService.isSpider(req)); req.setAgent("MSNBOT Is WaTching you"); assertFalse("'msnbot' matched pattern", spiderDetectorService.isSpider(req)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " did not match DNS patterns", spiderDetectorService.isSpider(req)); candidate = "baiduspiDer-dSPace-test.crawl.baIDu.com"; req.setRemoteHost(candidate); assertFalse(candidate + " matched DNS patterns", spiderDetectorService.isSpider(req)); }