SpiderDetector { public static boolean isSpider(String clientIP, String proxyIPs, String hostname, String agent) { return spiderDetectorService.isSpider(clientIP, proxyIPs, hostname, agent); } private  SpiderDetector(); static Set<String> getSpiderIpAddresses(); static Set<String> readPatterns(File patternFile); static boolean isSpider(String clientIP, String proxyIPs,
                                   String hostname, String agent); static boolean isSpider(HttpServletRequest request); static boolean isSpider(String ip);  }
@Test public void testIsSpiderHttpServletRequest() { DummyHttpServletRequest req = new DummyHttpServletRequest(); req.setAddress(NOT_A_BOT_ADDRESS); req.setRemoteHost("notabot.example.com"); req.setAgent("Firefox"); String candidate; req.setAgent("msnbot is watching you"); assertTrue("'msnbot' did not match any pattern", SpiderDetector.isSpider(req)); req.setAgent("Firefox"); assertFalse("'Firefox' matched a pattern", SpiderDetector.isSpider(req)); candidate = "192.168.2.1"; req.setAddress(candidate); assertTrue(candidate + " did not match IP patterns", SpiderDetector.isSpider(req)); req.setAddress(NOT_A_BOT_ADDRESS); assertFalse(NOT_A_BOT_ADDRESS + " matched IP patterns", SpiderDetector.isSpider(req)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; req.setRemoteHost(candidate); assertTrue(candidate + " did not match DNS patterns", SpiderDetector.isSpider(req)); candidate = "wiki.dspace.org"; req.setRemoteHost(candidate); assertFalse(candidate + " matched DNS patterns", SpiderDetector.isSpider(req)); }
@Test public void testIsSpiderStringStringStringString() { String candidate; candidate = "192.168.2.1"; assertTrue(candidate + " did not match IP patterns", SpiderDetector.isSpider(candidate, null, null, null)); candidate = NOT_A_BOT_ADDRESS; assertFalse(candidate + " matched IP patterns", SpiderDetector.isSpider(candidate, null, null, null)); candidate = "baiduspider-dspace-test.crawl.baidu.com"; assertTrue(candidate + " did not match DNS patterns", SpiderDetector.isSpider(NOT_A_BOT_ADDRESS, null, candidate, null)); candidate = "wiki.dspace.org"; assertFalse(candidate + " matched DNS patterns", SpiderDetector.isSpider(NOT_A_BOT_ADDRESS, null, candidate, null)); candidate = "msnbot is watching you"; assertTrue("'" + candidate + "' did not match agent patterns", SpiderDetector.isSpider(NOT_A_BOT_ADDRESS, null, null, candidate)); candidate = "Firefox"; assertFalse("'" + candidate + "' matched agent patterns", SpiderDetector.isSpider(NOT_A_BOT_ADDRESS, null, null, candidate)); }
@Test public void testIsSpiderString() { String candidate; candidate = "192.168.2.1"; assertTrue(candidate + " did not match IP patterns", SpiderDetector.isSpider(candidate, null, null, null)); candidate = NOT_A_BOT_ADDRESS; assertFalse(candidate + " matched IP patterns", SpiderDetector.isSpider(candidate, null, null, null)); }