UrlAnalyzer { public void processMetadata(Element element, AbstractMetadata md) throws org.jdom.JDOMException { SchemaPlugin schemaPlugin = schemaManager.getSchema(md.getDataInfo().getSchemaId()).getSchemaPlugin(); if (schemaPlugin instanceof LinkAwareSchemaPlugin) { metadataLinkRepository .findAll(metadatalinksTargetting(md)) .stream() .forEach(metadatalink -> { metadatalink.getLink().getRecords().remove(metadatalink); }); entityManager.flush(); ((LinkAwareSchemaPlugin) schemaPlugin).createLinkStreamer(new ILinkBuilder<Link, AbstractMetadata>() { @Override public Link found(String url) { Link link = linkRepository.findOneByUrl(url); if (link != null) { return link; } else { link = new Link(); link.setUrl(url); linkRepository.save(link); return link; } } @Override public void persist(Link link, AbstractMetadata metadata) { MetadataLink metadataLink = new MetadataLink(); metadataLink.setMetadataId(new Integer(metadata.getId())); metadataLink.setMetadataUuid(metadata.getUuid()); metadataLink.setLink(link); link.getRecords().add(metadataLink); linkRepository.save(link); } }).processAllRawText(element, md); entityManager.flush(); } }  void processMetadata(Element element, AbstractMetadata md); void purgeMetataLink(Link link); void deleteAll(); void testLink(Link link);  }
@Test public void encounteringAnUrlForTheFirstTimeAndPersistingIt() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata md = insertMetadataInDb(mdAsXml); UrlAnalyzer toTest = createToTest(); toTest.processMetadata(mdAsXml, md); Set<String> urlFromDb = linkRepository.findAll().stream().map(Link::getUrl).collect(Collectors.toSet()); assertTrue(urlFromDb.contains("HTTPS: assertTrue(urlFromDb.contains("ftp: assertTrue(urlFromDb.contains("http: assertTrue(urlFromDb.contains("http: assertTrue(urlFromDb.contains("http: assertTrue(urlFromDb.contains("http: assertEquals(6, urlFromDb.size()); SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository<MetadataLink, Integer>(MetadataLink.class, entityManager); List<MetadataLink> metadataLinkList = metadataLinkRepository.findAll(); assertEquals( metadataLinkList.stream().map(x -> x.getLink().getId()).collect(Collectors.toSet()), linkRepository.findAll().stream().map(Link::getId).collect(Collectors.toSet())); assertEquals( metadataLinkList.stream().map(x -> x.getMetadataId()).collect(Collectors.toSet()), Collections.singleton(md.getId())); }
@Test public void encounteringSameUrlInVariousMd() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata mdOne = insertMetadataInDb(mdAsXml); AbstractMetadata mdTwo = insertMetadataInDb(mdAsXml); UrlAnalyzer toTest = createToTest(); toTest.processMetadata(mdAsXml, mdOne); toTest.processMetadata(mdAsXml, mdTwo); Set<String> urlFromDb = linkRepository.findAll().stream().map(Link::getUrl).collect(Collectors.toSet()); assertEquals(6, urlFromDb.size()); SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository<MetadataLink, Integer>(MetadataLink.class, entityManager); List<MetadataLink> metadataLinkList = metadataLinkRepository.findAll(); assertEquals( metadataLinkList.stream().map(x -> x.getLink().getId()).collect(Collectors.toSet()), linkRepository.findAll().stream().map(Link::getId).collect(Collectors.toSet())); assertEquals( metadataLinkList.stream().map(x -> x.getMetadataId()).collect(Collectors.toSet()), Stream.of(mdOne.getId(), mdTwo.getId()).collect(Collectors.toSet())); assertEquals(12, metadataLinkList.size()); }
@Test public void doNotInsertSameLinkOrUrlTwice() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata md = insertMetadataInDb(mdAsXml); Xml.selectElement(mdAsXml, ". .setText("http: UrlAnalyzer toTest = createToTest(); toTest.processMetadata(mdAsXml, md); List<Link> urlFromDb = linkRepository.findAll(); assertEquals(5, urlFromDb.size()); List<MetadataLink> metadataLinkList = metadataLinkRepository.findAll(); assertEquals(5, metadataLinkList.size()); }
@Test public void cascadeDontFordbidMetadalinkDeletion() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata md = insertMetadataInDb(mdAsXml); UrlAnalyzer toTest = createToTest(); toTest.processMetadata(mdAsXml, md); toTest.processMetadata(mdAsXml, md); List<Link> urlFromDb = linkRepository.findAll(); assertEquals(6, urlFromDb.size()); List<MetadataLink> metadataLinkList = metadataLinkRepository.findAll(); assertEquals(6, metadataLinkList.size()); }
@Test public void deleteMetadaLinkWhenMdDisappeared() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata md = insertMetadataInDb(mdAsXml); UrlAnalyzer toTest = createToTest(); toTest.processMetadata(mdAsXml, md); SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository<MetadataLink, Integer>(MetadataLink.class, entityManager); List<MetadataLink> metadataLinkList = metadataLinkRepository.findAll(); assertEquals(6, metadataLinkList.size()); dataManager.deleteMetadata(context, md.getId() + ""); linkRepository.findAll().stream().forEach(toTest::purgeMetataLink); metadataLinkList = metadataLinkRepository.findAll(); assertEquals(0, metadataLinkList.size()); Set<String> urlFromDb = linkRepository.findAll().stream().map(Link::getUrl).collect(Collectors.toSet()); assertEquals(6, urlFromDb.size()); }
@Test public void updateMdCascade() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata md = insertMetadataInDb(mdAsXml); UrlAnalyzer toTest = createToTest(); toTest.processMetadata(mdAsXml, md); SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository<MetadataLink, Integer>(MetadataLink.class, entityManager); List<MetadataLink> metadataLinkList = metadataLinkRepository.findAll(); assertEquals(6, metadataLinkList.size()); Xml.selectElement(mdAsXml, ". toTest.processMetadata(mdAsXml, md); metadataLinkList = metadataLinkRepository.findAll(); assertEquals(5, metadataLinkList.size()); Set<String> urlFromDb = linkRepository.findAll().stream().map(Link::getUrl).collect(Collectors.toSet()); assertEquals(7, urlFromDb.size()); }
@Test public void urlCheckerToSetStatus() throws Exception { Element mdAsXml = getMdAsXml(); AbstractMetadata md = insertMetadataInDb(mdAsXml); UrlAnalyzer toTest = createToTest(); UrlChecker mockUrlChecker = mock(UrlChecker.class); toTest.urlChecker = mockUrlChecker; Mockito.when(mockUrlChecker.getUrlStatus(eq("http: Mockito.when(mockUrlChecker.getUrlStatus(eq("http: Mockito.when(mockUrlChecker.getUrlStatus(eq("ftp: Mockito.when(mockUrlChecker.getUrlStatus(eq("HTTPS: Mockito.when(mockUrlChecker.getUrlStatus(contains("aquastat"))).thenReturn(new LinkStatus().setStatusValue("418").setStatusInfo("I am teapot").setFailing(true)); Mockito.when(mockUrlChecker.getUrlStatus(contains("data.fao"))).thenReturn(new LinkStatus().setStatusValue("418").setStatusInfo("I am teapot").setFailing(true)); toTest.processMetadata(mdAsXml, md); SimpleJpaRepository statusRepository = new SimpleJpaRepository<LinkStatus, Integer>(LinkStatus.class, entityManager); assertEquals(0, statusRepository.findAll().size()); linkRepository.findAll().stream().forEach(toTest::testLink); List<LinkStatus> allStatus = statusRepository.findAll(); assertEquals(6, allStatus.size()); assertEquals(Stream.of("200", "406", "418").collect(Collectors.toSet()), allStatus.stream().map(LinkStatus::getStatusValue).collect(Collectors.toSet())); toTest.urlChecker = mockUrlChecker; Mockito.when(mockUrlChecker.getUrlStatus(eq("http: Mockito.when(mockUrlChecker.getUrlStatus(eq("http: Mockito.when(mockUrlChecker.getUrlStatus(eq("ftp: Mockito.when(mockUrlChecker.getUrlStatus(eq("HTTPS: Mockito.when(mockUrlChecker.getUrlStatus(contains("aquastat"))).thenReturn(new LinkStatus().setStatusValue("418").setStatusInfo("I am teapot").setFailing(true)); Mockito.when(mockUrlChecker.getUrlStatus(contains("data.fao"))).thenReturn(new LinkStatus().setStatusValue("418").setStatusInfo("I am teapot").setFailing(true)); linkRepository.findAll().stream().forEach(toTest::testLink); allStatus = statusRepository.findAll(); assertEquals(12, allStatus.size()); Map<String, Long> grouped = allStatus.stream().map(LinkStatus::getStatusValue).collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); assertEquals(6L, grouped.get("200").longValue()); assertEquals(1L,grouped.get("406").longValue()); assertEquals(5L, grouped.get("418").longValue()); }