CrawlingStrategy { public boolean isCrawable(URL url, int depth) { if (url != null && !visitedUrls.contains(url) && isCrawlingAllowed(url) && depth < crawlerMaxDepth) { visitedUrls.add(url); if(url.getPath().endsWith("/")){ try { visitedUrls.add(new URL(url.toString().substring(0, url.toString().length() -1))); } catch (MalformedURLException ex) { Logger.getLogger(CrawlingStrategy.class.getName()).log(Level.SEVERE, null, ex); } } return true; } return false; } CrawlingStrategy(int crawlerMaxDepth);  CrawlingStrategy(int crawlerMaxDepth, List<URL> visitedUrls, Map<String, Collection<String>> rootUrlsPermissions); boolean isCrawable(URL url, int depth);  }
@Test public void testIsCrawable() { try { URL url = new URL("http: assertTrue(strategy.isCrawable(url, 2)); } catch (MalformedURLException ex) { Logger.getLogger(CrawlingStrategyTest.class.getName()).log(Level.SEVERE, null, ex); fail("Failed to instanciate the url tou test."); } }
@Test public void testExceedDepthPagesNotCrawable() { try { URL url = new URL("http: assertFalse(strategy.isCrawable(url, 11)); } catch (MalformedURLException ex) { Logger.getLogger(CrawlingStrategyTest.class.getName()).log(Level.SEVERE, null, ex); fail("Failed to instanciate the url tou test."); } }
@Test public void testAlreadyVisitedPagesNotCrawable() { try { URL url = new URL("http: assertTrue(strategy.isCrawable(url, 2)); } catch (MalformedURLException ex) { Logger.getLogger(CrawlingStrategyTest.class.getName()).log(Level.SEVERE, null, ex); fail("Failed to instanciate the url tou test."); } }
@Test public void testRobotExclusionsNotCrawable() { try { URL url = new URL("http: assertFalse(strategy.isCrawable(url, 2)); } catch (MalformedURLException ex) { Logger.getLogger(CrawlingStrategyTest.class.getName()).log(Level.SEVERE, null, ex); fail("Failed to instanciate the url tou test."); } }