FrontierImpl implements Frontier { @Override public void crawlingDone(List<CrawleableUri> uris) { LOGGER.info("One worker finished his work and crawled " + uris.size() + " URIs."); if (queue instanceof BlockingQueue) { ((BlockingQueue<?>) queue).markUrisAsAccessible(uris); } for (CrawleableUri uri : uris) { Long recrawlOn = (Long) uri.getData(Constants.URI_PREFERRED_RECRAWL_ON); if ((recrawlOn != null) && (recrawlOn < System.currentTimeMillis())) { CrawleableUri recrawlUri = new CrawleableUri(uri.getUri(), uri.getIpAddress()); recrawlUri.addData(Constants.URI_TYPE_KEY, uri.getData(Constants.URI_TYPE_KEY)); addNewUri(recrawlUri); } else { uriFilter.getKnownUriFilter().add(uri, System.currentTimeMillis()); } } } FrontierImpl(UriNormalizer normalizer, UriFilterComposer relationalUriFilter, UriQueue queue,
			List<UriGenerator> uriGenerators, GraphLogger graphLogger, boolean doesRecrawling, long generalRecrawlTime,
			long timerPeriod);  FrontierImpl(UriNormalizer normalizer, UriFilterComposer relationalUriFilter, UriQueue queue,
			List<UriGenerator> uriGenerators, boolean doesRecrawling, long generalRecrawlTime, long timerPeriod);  FrontierImpl(UriNormalizer normalizer, UriFilterComposer relationalUriFilter, URIReferences uriReferences,
			UriQueue queue, List<UriGenerator> uriGenerators,boolean doesRecrawling, OutDatedUriRetriever outDatedUriRetriever);  FrontierImpl(UriNormalizer normalizer, UriFilterComposer relationalUriFilter, UriQueue queue,
			List<UriGenerator> uriGenerators, boolean doesRecrawling);  FrontierImpl(UriNormalizer normalizer, UriFilterComposer relationalUriFilter, UriQueue queue,
			List<UriGenerator> uriGenerators);  FrontierImpl(UriNormalizer normalizer, UriFilterComposer relationalUriFilter, URIReferences uriReferences,
			UriQueue queue, List<UriGenerator> uriGenerators, GraphLogger graphLogger, boolean doesRecrawling,
			long generalRecrawlTime, long timerPeriod,OutDatedUriRetriever outDatedUriRetriever); @Override List<CrawleableUri> getNextUris(); @Override void addNewUris(List<CrawleableUri> uris); @Override void addNewUri(CrawleableUri uri); @Override void crawlingDone(List<CrawleableUri> uris); @Override int getNumberOfPendingUris(); @Override boolean doesRecrawling(); @Override void close(); static long getGeneralRecrawlTime(); UriQueue getQueue(); static final long DEFAULT_GENERAL_RECRAWL_TIME; }
@Test public void crawlingDone() throws Exception { List<CrawleableUri> crawledUris = new ArrayList<>(); CrawleableUri uri_1 = cuf.create(new URI("http: InetAddress.getByName("127.0.0.1"), UriType.DEREFERENCEABLE); CrawleableUri uri_2 = cuf.create(new URI("http: InetAddress.getByName("127.0.0.1"), UriType.DEREFERENCEABLE); crawledUris.add(uri_1); crawledUris.add(uri_2); frontier.crawlingDone(crawledUris); assertFalse("uri_1 has been already crawled", frontier.uriFilter.isUriGood(uri_1)); }