@Test public void importRecordsFromZip() throws StudyImporterException, IOException { URL resource = getClass().getResource("/org/globalbioticinteractions/dataset/dwca.zip"); DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null); DatasetImpl dataset = new DatasetImpl("some/namespace", URI.create("file: JsonNode jsonNode = new ObjectMapper().readTree("{ " + "\"interactionTypeId\": \"http: "\"url\": \"" + resource.toExternalForm() + "\"" + "}"); dataset.setConfig(jsonNode); studyImporterForDwCA.setDataset(dataset); String expectedCitation = dataset.getCitation(); AtomicBoolean someRecords = new AtomicBoolean(false); studyImporterForDwCA.setInteractionListener(new InteractionListener() { @Override public void newLink(Map<String, String> link) throws StudyImporterException { String associatedTaxa = link.get("http: String dynamicProperties = link.get("http: assertThat(StringUtils.isNotBlank(associatedTaxa) || StringUtils.isNotBlank(dynamicProperties), is(true)); assertThat(link.get(TaxonUtil.SOURCE_TAXON_NAME), is(not(nullValue()))); assertThat(link.get(TaxonUtil.TARGET_TAXON_NAME), is(not(nullValue()))); assertThat(link.get(INTERACTION_TYPE_NAME), is(not(nullValue()))); assertThat(link.get(INTERACTION_TYPE_ID), is(not(nullValue()))); assertThat(link.get(DatasetImporterForTSV.STUDY_SOURCE_CITATION), containsString(expectedCitation)); assertThat(link.get(REFERENCE_ID), startsWith("https: assertThat(link.get(DatasetImporterForTSV.REFERENCE_CITATION), startsWith("https: assertThat(link.get(REFERENCE_URL), startsWith("https: someRecords.set(true); } }); studyImporterForDwCA.importStudy(); assertThat(someRecords.get(), is(true)); }
@Override public void importStudy() throws StudyImporterException { URI archiveURI = getDataset().getArchiveURI(); Path tmpDwA = null; try { if (getDataset() == null) { throw new IllegalArgumentException("no dataset found"); } String archiveURL = getDataset().getOrDefault("url", archiveURI == null ? null : archiveURI.toString()); getLogger().info(null, "[" + archiveURL + "]: indexing interaction records"); File dwcaFile = null; try { URI dwcaURI = URI.create(archiveURL); tmpDwA = Files.createTempDirectory("dwca"); Archive archive; if (CacheUtil.isLocalDir(dwcaURI)) { archive = DwCAUtil.archiveFor(dwcaURI, tmpDwA.toString()); } else { dwcaFile = File.createTempFile("dwca", "tmp.zip"); FileUtils.copyToFile(getDataset().retrieve(dwcaURI), dwcaFile); dwcaFile.deleteOnExit(); archive = DwCAUtil.archiveFor(dwcaFile.toURI(), tmpDwA.toString()); } InteractionListenerWithInteractionTypeMapping listenerProxy = new InteractionListenerWithInteractionTypeMapping( new InteractionListenerWithContext(), InteractUtil.createInteractionTypeMapperForImporter(getDataset()), getLogger()); importResourceRelationExtension(archive, listenerProxy); importAssociatedTaxaExtension(archive, listenerProxy); int i = importCore(archive, listenerProxy); getLogger().info(null, "[" + archiveURL + "]: scanned [" + i + "] record(s)"); getLogger().info(null, "[" + archiveURL + "]: detected [" + listenerProxy.getNumberOfSubmittedLinks() + "] interaction record(s)"); } finally { if (dwcaFile != null && dwcaFile.exists() && dwcaFile.isFile()) { FileUtils.deleteQuietly(dwcaFile); } } } catch (IOException | IllegalStateException e) { throw new StudyImporterException("failed to read archive [" + archiveURI + "]", e); } finally { if (tmpDwA != null) { org.apache.commons.io.FileUtils.deleteQuietly(tmpDwA.toFile()); } } }
DatasetImporterForDwCA extends DatasetImporterWithListener { @Override public void importStudy() throws StudyImporterException { URI archiveURI = getDataset().getArchiveURI(); Path tmpDwA = null; try { if (getDataset() == null) { throw new IllegalArgumentException("no dataset found"); } String archiveURL = getDataset().getOrDefault("url", archiveURI == null ? null : archiveURI.toString()); getLogger().info(null, "[" + archiveURL + "]: indexing interaction records"); File dwcaFile = null; try { URI dwcaURI = URI.create(archiveURL); tmpDwA = Files.createTempDirectory("dwca"); Archive archive; if (CacheUtil.isLocalDir(dwcaURI)) { archive = DwCAUtil.archiveFor(dwcaURI, tmpDwA.toString()); } else { dwcaFile = File.createTempFile("dwca", "tmp.zip"); FileUtils.copyToFile(getDataset().retrieve(dwcaURI), dwcaFile); dwcaFile.deleteOnExit(); archive = DwCAUtil.archiveFor(dwcaFile.toURI(), tmpDwA.toString()); } InteractionListenerWithInteractionTypeMapping listenerProxy = new InteractionListenerWithInteractionTypeMapping( new InteractionListenerWithContext(), InteractUtil.createInteractionTypeMapperForImporter(getDataset()), getLogger()); importResourceRelationExtension(archive, listenerProxy); importAssociatedTaxaExtension(archive, listenerProxy); int i = importCore(archive, listenerProxy); getLogger().info(null, "[" + archiveURL + "]: scanned [" + i + "] record(s)"); getLogger().info(null, "[" + archiveURL + "]: detected [" + listenerProxy.getNumberOfSubmittedLinks() + "] interaction record(s)"); } finally { if (dwcaFile != null && dwcaFile.exists() && dwcaFile.isFile()) { FileUtils.deleteQuietly(dwcaFile); } } } catch (IOException | IllegalStateException e) { throw new StudyImporterException("failed to read archive [" + archiveURI + "]", e); } finally { if (tmpDwA != null) { org.apache.commons.io.FileUtils.deleteQuietly(tmpDwA.toFile()); } } } }
DatasetImporterForDwCA extends DatasetImporterWithListener { @Override public void importStudy() throws StudyImporterException { URI archiveURI = getDataset().getArchiveURI(); Path tmpDwA = null; try { if (getDataset() == null) { throw new IllegalArgumentException("no dataset found"); } String archiveURL = getDataset().getOrDefault("url", archiveURI == null ? null : archiveURI.toString()); getLogger().info(null, "[" + archiveURL + "]: indexing interaction records"); File dwcaFile = null; try { URI dwcaURI = URI.create(archiveURL); tmpDwA = Files.createTempDirectory("dwca"); Archive archive; if (CacheUtil.isLocalDir(dwcaURI)) { archive = DwCAUtil.archiveFor(dwcaURI, tmpDwA.toString()); } else { dwcaFile = File.createTempFile("dwca", "tmp.zip"); FileUtils.copyToFile(getDataset().retrieve(dwcaURI), dwcaFile); dwcaFile.deleteOnExit(); archive = DwCAUtil.archiveFor(dwcaFile.toURI(), tmpDwA.toString()); } InteractionListenerWithInteractionTypeMapping listenerProxy = new InteractionListenerWithInteractionTypeMapping( new InteractionListenerWithContext(), InteractUtil.createInteractionTypeMapperForImporter(getDataset()), getLogger()); importResourceRelationExtension(archive, listenerProxy); importAssociatedTaxaExtension(archive, listenerProxy); int i = importCore(archive, listenerProxy); getLogger().info(null, "[" + archiveURL + "]: scanned [" + i + "] record(s)"); getLogger().info(null, "[" + archiveURL + "]: detected [" + listenerProxy.getNumberOfSubmittedLinks() + "] interaction record(s)"); } finally { if (dwcaFile != null && dwcaFile.exists() && dwcaFile.isFile()) { FileUtils.deleteQuietly(dwcaFile); } } } catch (IOException | IllegalStateException e) { throw new StudyImporterException("failed to read archive [" + archiveURI + "]", e); } finally { if (tmpDwA != null) { org.apache.commons.io.FileUtils.deleteQuietly(tmpDwA.toFile()); } } } DatasetImporterForDwCA(ParserFactory parserFactory, NodeFactory nodeFactory); }
DatasetImporterForDwCA extends DatasetImporterWithListener { @Override public void importStudy() throws StudyImporterException { URI archiveURI = getDataset().getArchiveURI(); Path tmpDwA = null; try { if (getDataset() == null) { throw new IllegalArgumentException("no dataset found"); } String archiveURL = getDataset().getOrDefault("url", archiveURI == null ? null : archiveURI.toString()); getLogger().info(null, "[" + archiveURL + "]: indexing interaction records"); File dwcaFile = null; try { URI dwcaURI = URI.create(archiveURL); tmpDwA = Files.createTempDirectory("dwca"); Archive archive; if (CacheUtil.isLocalDir(dwcaURI)) { archive = DwCAUtil.archiveFor(dwcaURI, tmpDwA.toString()); } else { dwcaFile = File.createTempFile("dwca", "tmp.zip"); FileUtils.copyToFile(getDataset().retrieve(dwcaURI), dwcaFile); dwcaFile.deleteOnExit(); archive = DwCAUtil.archiveFor(dwcaFile.toURI(), tmpDwA.toString()); } InteractionListenerWithInteractionTypeMapping listenerProxy = new InteractionListenerWithInteractionTypeMapping( new InteractionListenerWithContext(), InteractUtil.createInteractionTypeMapperForImporter(getDataset()), getLogger()); importResourceRelationExtension(archive, listenerProxy); importAssociatedTaxaExtension(archive, listenerProxy); int i = importCore(archive, listenerProxy); getLogger().info(null, "[" + archiveURL + "]: scanned [" + i + "] record(s)"); getLogger().info(null, "[" + archiveURL + "]: detected [" + listenerProxy.getNumberOfSubmittedLinks() + "] interaction record(s)"); } finally { if (dwcaFile != null && dwcaFile.exists() && dwcaFile.isFile()) { FileUtils.deleteQuietly(dwcaFile); } } } catch (IOException | IllegalStateException e) { throw new StudyImporterException("failed to read archive [" + archiveURI + "]", e); } finally { if (tmpDwA != null) { org.apache.commons.io.FileUtils.deleteQuietly(tmpDwA.toFile()); } } } DatasetImporterForDwCA(ParserFactory parserFactory, NodeFactory nodeFactory); @Override void importStudy(); int importCore(Archive archive, InteractionListener interactionListener); void handleRecord(InteractionListener interactionListener, Record rec); static void addRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static Map<String, String> parseRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(String occurrenceRemarks); void addUSNMStyleHostOccurrenceRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static void mapCoreProperties(Record rec, Map<String, String> interactionProperties); static void mapLocationAndReferenceInfo(Record rec, Map<String, String> interactionProperties); static void mapSourceProperties(Record rec, Map<String, String> interactionProperties); static void mapIfAvailable(Record rec, Map<String, String> interactionProperties, String key, Term term); static void mapIfAvailable(Map<String, String> interactionProperties, String key, String value); static Map<String, String> parseUSNMStyleHostOccurrenceRemarks(String occurrenceRemarks); static Map<String, String> parseJsonChunk(String candidateJsonChunk); }
DatasetImporterForDwCA extends DatasetImporterWithListener { @Override public void importStudy() throws StudyImporterException { URI archiveURI = getDataset().getArchiveURI(); Path tmpDwA = null; try { if (getDataset() == null) { throw new IllegalArgumentException("no dataset found"); } String archiveURL = getDataset().getOrDefault("url", archiveURI == null ? null : archiveURI.toString()); getLogger().info(null, "[" + archiveURL + "]: indexing interaction records"); File dwcaFile = null; try { URI dwcaURI = URI.create(archiveURL); tmpDwA = Files.createTempDirectory("dwca"); Archive archive; if (CacheUtil.isLocalDir(dwcaURI)) { archive = DwCAUtil.archiveFor(dwcaURI, tmpDwA.toString()); } else { dwcaFile = File.createTempFile("dwca", "tmp.zip"); FileUtils.copyToFile(getDataset().retrieve(dwcaURI), dwcaFile); dwcaFile.deleteOnExit(); archive = DwCAUtil.archiveFor(dwcaFile.toURI(), tmpDwA.toString()); } InteractionListenerWithInteractionTypeMapping listenerProxy = new InteractionListenerWithInteractionTypeMapping( new InteractionListenerWithContext(), InteractUtil.createInteractionTypeMapperForImporter(getDataset()), getLogger()); importResourceRelationExtension(archive, listenerProxy); importAssociatedTaxaExtension(archive, listenerProxy); int i = importCore(archive, listenerProxy); getLogger().info(null, "[" + archiveURL + "]: scanned [" + i + "] record(s)"); getLogger().info(null, "[" + archiveURL + "]: detected [" + listenerProxy.getNumberOfSubmittedLinks() + "] interaction record(s)"); } finally { if (dwcaFile != null && dwcaFile.exists() && dwcaFile.isFile()) { FileUtils.deleteQuietly(dwcaFile); } } } catch (IOException | IllegalStateException e) { throw new StudyImporterException("failed to read archive [" + archiveURI + "]", e); } finally { if (tmpDwA != null) { org.apache.commons.io.FileUtils.deleteQuietly(tmpDwA.toFile()); } } } DatasetImporterForDwCA(ParserFactory parserFactory, NodeFactory nodeFactory); @Override void importStudy(); int importCore(Archive archive, InteractionListener interactionListener); void handleRecord(InteractionListener interactionListener, Record rec); static void addRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static Map<String, String> parseRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(String occurrenceRemarks); void addUSNMStyleHostOccurrenceRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static void mapCoreProperties(Record rec, Map<String, String> interactionProperties); static void mapLocationAndReferenceInfo(Record rec, Map<String, String> interactionProperties); static void mapSourceProperties(Record rec, Map<String, String> interactionProperties); static void mapIfAvailable(Record rec, Map<String, String> interactionProperties, String key, Term term); static void mapIfAvailable(Map<String, String> interactionProperties, String key, String value); static Map<String, String> parseUSNMStyleHostOccurrenceRemarks(String occurrenceRemarks); static Map<String, String> parseJsonChunk(String candidateJsonChunk); static final String EXTENSION_ASSOCIATED_TAXA; static final String EXTENSION_RESOURCE_RELATIONSHIP; static final String EXTENSION_TAXON; static final Pattern EX_NOTATION; static final Pattern REARED_EX_NOTATION; static final Pattern PATTERN_ASSOCIATED_TAXA_IDEA; static final Pattern PATTERN_ASSOCIATED_TAXA_EAE; }