@Test public void testAddSimpleUrls() throws Exception { List<String> urls = Arrays.asList("someURL1", "someURL2", "someURL3"); service.addSimpleUrls(urls, "NEW", "site.com", "origin.com"); assertEquals(3, urlRepo.count()); Iterable<MongoSimpleURL> actual = urlRepo.find("{domain: #}", "site.com").as(MongoSimpleURL.class); List<String> urlsFound = new ArrayList<>(); for (MongoSimpleURL crtActual : actual) { assertEquals("site.com", crtActual.getDomain()); urlsFound.add(crtActual.getUrl()); } assertTrue(urlsFound.containsAll(urls)); }
@Override public void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName) { Iterable<MongoSimpleURL> old; Timer.Context time = findLinksTimer.time(); try { old = urlRepo.find("{url: {$in: #}}", urls).projection("{'_id': 0, 'url': 1}").as(MongoSimpleURL.class); } finally { time.stop(); } for (MongoSimpleURL s : old) { urls.remove(s.getUrl()); } List<MongoSimpleURL> objs = new ArrayList<>(); long dtm = System.currentTimeMillis(); for (String url : urls) { objs.add(new MongoSimpleURL(url, url, domainURL, sourceDomainName, status, 0, dtm)); } urlRepo.insert(objs.toArray()); }
MongoUrlServiceImpl implements UrlService { @Override public void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName) { Iterable<MongoSimpleURL> old; Timer.Context time = findLinksTimer.time(); try { old = urlRepo.find("{url: {$in: #}}", urls).projection("{'_id': 0, 'url': 1}").as(MongoSimpleURL.class); } finally { time.stop(); } for (MongoSimpleURL s : old) { urls.remove(s.getUrl()); } List<MongoSimpleURL> objs = new ArrayList<>(); long dtm = System.currentTimeMillis(); for (String url : urls) { objs.add(new MongoSimpleURL(url, url, domainURL, sourceDomainName, status, 0, dtm)); } urlRepo.insert(objs.toArray()); } }
MongoUrlServiceImpl implements UrlService { @Override public void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName) { Iterable<MongoSimpleURL> old; Timer.Context time = findLinksTimer.time(); try { old = urlRepo.find("{url: {$in: #}}", urls).projection("{'_id': 0, 'url': 1}").as(MongoSimpleURL.class); } finally { time.stop(); } for (MongoSimpleURL s : old) { urls.remove(s.getUrl()); } List<MongoSimpleURL> objs = new ArrayList<>(); long dtm = System.currentTimeMillis(); for (String url : urls) { objs.add(new MongoSimpleURL(url, url, domainURL, sourceDomainName, status, 0, dtm)); } urlRepo.insert(objs.toArray()); }  }
MongoUrlServiceImpl implements UrlService { @Override public void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName) { Iterable<MongoSimpleURL> old; Timer.Context time = findLinksTimer.time(); try { old = urlRepo.find("{url: {$in: #}}", urls).projection("{'_id': 0, 'url': 1}").as(MongoSimpleURL.class); } finally { time.stop(); } for (MongoSimpleURL s : old) { urls.remove(s.getUrl()); } List<MongoSimpleURL> objs = new ArrayList<>(); long dtm = System.currentTimeMillis(); for (String url : urls) { objs.add(new MongoSimpleURL(url, url, domainURL, sourceDomainName, status, 0, dtm)); } urlRepo.insert(objs.toArray()); }  @PostConstruct void init(); DomainURL addDomainUrl(String domainName, String domainUrl, long coolDownPeriod); @Override void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName); @Override void addDomainLinks(List<DomainLink> domainLinks); @Override List<DomainURL> findDomains(Pageable pageable); @Override Collection<SimpleURL> findURLs(String address, String status, int pageNo, int pageSize); @Override Collection<SimpleURL> findExternalURLs(String address, String status, int pageNo, int pageSize); void removeSimpleUrl(String url); @Override void updateSimpleUrlsStatus(List<String> urls, String status); @Override void updateSimpleUrlErrorStatus(String url, int errorDelta); void removeDomainUrl(String domainURL); @Override void removeDomains(); @Override List<Entries> query(String query); @Override long countAllNodes(); @Override long countAllDomains(); @Override long countAllLinks(); @Override long countDomainLinks(String domainUrl); }
MongoUrlServiceImpl implements UrlService { @Override public void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName) { Iterable<MongoSimpleURL> old; Timer.Context time = findLinksTimer.time(); try { old = urlRepo.find("{url: {$in: #}}", urls).projection("{'_id': 0, 'url': 1}").as(MongoSimpleURL.class); } finally { time.stop(); } for (MongoSimpleURL s : old) { urls.remove(s.getUrl()); } List<MongoSimpleURL> objs = new ArrayList<>(); long dtm = System.currentTimeMillis(); for (String url : urls) { objs.add(new MongoSimpleURL(url, url, domainURL, sourceDomainName, status, 0, dtm)); } urlRepo.insert(objs.toArray()); }  @PostConstruct void init(); DomainURL addDomainUrl(String domainName, String domainUrl, long coolDownPeriod); @Override void addSimpleUrls(List<String> urls, String status, String domainURL, String sourceDomainName); @Override void addDomainLinks(List<DomainLink> domainLinks); @Override List<DomainURL> findDomains(Pageable pageable); @Override Collection<SimpleURL> findURLs(String address, String status, int pageNo, int pageSize); @Override Collection<SimpleURL> findExternalURLs(String address, String status, int pageNo, int pageSize); void removeSimpleUrl(String url); @Override void updateSimpleUrlsStatus(List<String> urls, String status); @Override void updateSimpleUrlErrorStatus(String url, int errorDelta); void removeDomainUrl(String domainURL); @Override void removeDomains(); @Override List<Entries> query(String query); @Override long countAllNodes(); @Override long countAllDomains(); @Override long countAllLinks(); @Override long countDomainLinks(String domainUrl); @Autowired
    public MetricsConfig metricsConfig; @Autowired
    public MongoConfig mongoConfig; public MongoCollection domainRepo; public MongoCollection urlRepo; }