@Test public void hasResourceRelationshipsOccurrenceToTaxa() throws IOException, URISyntaxException { URI sampleArchive = getClass().getResource("inaturalist-dwca-rr.zip").toURI(); Archive archive = DwCAUtil.archiveFor(sampleArchive, "target/tmp"); AtomicInteger numberOfFoundLinks = new AtomicInteger(0); importResourceRelationExtension(archive, new InteractionListener() { @Override public void newLink(Map<String, String> link) throws StudyImporterException { numberOfFoundLinks.incrementAndGet(); if (1 == numberOfFoundLinks.get()) { assertThat(link.get(TaxonUtil.SOURCE_TAXON_ID), is("http: assertThat(link.get(TaxonUtil.SOURCE_TAXON_NAME), is("Gorgonocephalus eucnemis")); assertThat(link.get(DatasetImporterForTSV.SOURCE_OCCURRENCE_ID), is("http: assertThat(link.get(INTERACTION_TYPE_NAME), is("Eaten by")); assertThat(link.get(INTERACTION_TYPE_ID), is("http: assertThat(link.get(DatasetImporterForTSV.BASIS_OF_RECORD_NAME), is("HumanObservation")); assertThat(link.get(TaxonUtil.TARGET_TAXON_ID), is("http: assertThat(link.get(TaxonUtil.TARGET_TAXON_NAME), is("Enhydra lutris kenyoni")); assertThat(link.get(DatasetImporterForTSV.REFERENCE_CITATION), is("https: } } }); assertThat(numberOfFoundLinks.get(), is(1)); }
static void importResourceRelationExtension(Archive archive, InteractionListener interactionListener) { ArchiveFile resourceExtension = findResourceRelationshipExtension(archive); if (resourceExtension != null) { DB db = DBMaker .newMemoryDirectDB() .compressionEnable() .transactionDisable() .make(); final HTreeMap<String, Map<String, Map<String, String>>> termIdPropMap = db .createHashMap("termIdPropMap") .make(); final Set<String> referencedSourceIds = db .createHashSet("sourceIdMap") .make(); final Set<String> referencedTargetIds = db .createHashSet("targetIdMap") .make(); for (Record record : resourceExtension) { String targetId = record.value(DwcTerm.relatedResourceID); String sourceId = record.value(DwcTerm.resourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { referencedSourceIds.add(sourceId); referencedTargetIds.add(targetId); } } final List<DwcTerm> idTerms = Arrays.asList( DwcTerm.occurrenceID, DwcTerm.taxonID); List<ArchiveFile> archiveFiles = new ArrayList<>(); archiveFiles.add(archive.getCore()); ArchiveFile taxon = findResourceExtension(archive, EXTENSION_TAXON); if (taxon != null) { archiveFiles.add(taxon); } for (ArchiveFile archiveFile : archiveFiles) { for (Record record : archiveFile) { for (DwcTerm idTerm : idTerms) { attemptLinkUsingTerm(termIdPropMap, referencedSourceIds, referencedTargetIds, record, idTerm); } } } for (Record record : resourceExtension) { Map<String, String> props = new TreeMap<>(); String sourceId = record.value(DwcTerm.resourceID); String relationship = record.value(DwcTerm.relationshipOfResource); Optional<Term> relationshipOfResourceIDTerm = record .terms() .stream() .filter(x -> StringUtils.equals(x.simpleName(), "relationshipOfResourceID")) .findFirst(); String relationshipTypeIdValue = relationshipOfResourceIDTerm .map(record::value) .orElse(null); String targetId = record.value(DwcTerm.relatedResourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { appendVerbatimResourceRelationsValues(record, props); String relationshipAccordingTo = record.value(DwcTerm.relationshipAccordingTo); if (StringUtils.isNotBlank(relationshipAccordingTo)) { props.putIfAbsent(REFERENCE_CITATION, relationshipAccordingTo); } putIfAbsentAndNotBlank(props, INTERACTION_TYPE_NAME, relationship); putIfAbsentAndNotBlank(props, INTERACTION_TYPE_ID, relationshipTypeIdValue); putIfAbsentAndNotBlank(props, DatasetImporterForMetaTable.EVENT_DATE, record.value(DwcTerm.relationshipEstablishedDate)); for (DwcTerm idTerm : idTerms) { if (termIdPropMap.containsKey(idTerm.qualifiedName())) { Map<String, Map<String, String>> propMap = termIdPropMap.get(idTerm.qualifiedName()); Map<String, String> sourceIdProperties = propMap.get(sourceId); Pair<String, String> idLabelPair = Pair.of( SOURCE_OCCURRENCE_ID, TARGET_OCCURRENCE_ID); if (idTerm.equals(DwcTerm.taxonID)) { idLabelPair = Pair.of(SOURCE_TAXON_ID, TARGET_TAXON_ID); } populatePropertiesAssociatedWithId(props, sourceId, true, sourceIdProperties, idLabelPair); Map<String, String> targetIdProperties = propMap.get(targetId); populatePropertiesAssociatedWithId(props, targetId, false, targetIdProperties, idLabelPair); } } try { interactionListener.newLink(props); } catch (StudyImporterException e) { } } } } }
DatasetImporterForDwCA extends DatasetImporterWithListener { static void importResourceRelationExtension(Archive archive, InteractionListener interactionListener) { ArchiveFile resourceExtension = findResourceRelationshipExtension(archive); if (resourceExtension != null) { DB db = DBMaker .newMemoryDirectDB() .compressionEnable() .transactionDisable() .make(); final HTreeMap<String, Map<String, Map<String, String>>> termIdPropMap = db .createHashMap("termIdPropMap") .make(); final Set<String> referencedSourceIds = db .createHashSet("sourceIdMap") .make(); final Set<String> referencedTargetIds = db .createHashSet("targetIdMap") .make(); for (Record record : resourceExtension) { String targetId = record.value(DwcTerm.relatedResourceID); String sourceId = record.value(DwcTerm.resourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { referencedSourceIds.add(sourceId); referencedTargetIds.add(targetId); } } final List<DwcTerm> idTerms = Arrays.asList( DwcTerm.occurrenceID, DwcTerm.taxonID); List<ArchiveFile> archiveFiles = new ArrayList<>(); archiveFiles.add(archive.getCore()); ArchiveFile taxon = findResourceExtension(archive, EXTENSION_TAXON); if (taxon != null) { archiveFiles.add(taxon); } for (ArchiveFile archiveFile : archiveFiles) { for (Record record : archiveFile) { for (DwcTerm idTerm : idTerms) { attemptLinkUsingTerm(termIdPropMap, referencedSourceIds, referencedTargetIds, record, idTerm); } } } for (Record record : resourceExtension) { Map<String, String> props = new TreeMap<>(); String sourceId = record.value(DwcTerm.resourceID); String relationship = record.value(DwcTerm.relationshipOfResource); Optional<Term> relationshipOfResourceIDTerm = record .terms() .stream() .filter(x -> StringUtils.equals(x.simpleName(), "relationshipOfResourceID")) .findFirst(); String relationshipTypeIdValue = relationshipOfResourceIDTerm .map(record::value) .orElse(null); String targetId = record.value(DwcTerm.relatedResourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { appendVerbatimResourceRelationsValues(record, props); String relationshipAccordingTo = record.value(DwcTerm.relationshipAccordingTo); if (StringUtils.isNotBlank(relationshipAccordingTo)) { props.putIfAbsent(REFERENCE_CITATION, relationshipAccordingTo); } putIfAbsentAndNotBlank(props, INTERACTION_TYPE_NAME, relationship); putIfAbsentAndNotBlank(props, INTERACTION_TYPE_ID, relationshipTypeIdValue); putIfAbsentAndNotBlank(props, DatasetImporterForMetaTable.EVENT_DATE, record.value(DwcTerm.relationshipEstablishedDate)); for (DwcTerm idTerm : idTerms) { if (termIdPropMap.containsKey(idTerm.qualifiedName())) { Map<String, Map<String, String>> propMap = termIdPropMap.get(idTerm.qualifiedName()); Map<String, String> sourceIdProperties = propMap.get(sourceId); Pair<String, String> idLabelPair = Pair.of( SOURCE_OCCURRENCE_ID, TARGET_OCCURRENCE_ID); if (idTerm.equals(DwcTerm.taxonID)) { idLabelPair = Pair.of(SOURCE_TAXON_ID, TARGET_TAXON_ID); } populatePropertiesAssociatedWithId(props, sourceId, true, sourceIdProperties, idLabelPair); Map<String, String> targetIdProperties = propMap.get(targetId); populatePropertiesAssociatedWithId(props, targetId, false, targetIdProperties, idLabelPair); } } try { interactionListener.newLink(props); } catch (StudyImporterException e) { } } } } } }
DatasetImporterForDwCA extends DatasetImporterWithListener { static void importResourceRelationExtension(Archive archive, InteractionListener interactionListener) { ArchiveFile resourceExtension = findResourceRelationshipExtension(archive); if (resourceExtension != null) { DB db = DBMaker .newMemoryDirectDB() .compressionEnable() .transactionDisable() .make(); final HTreeMap<String, Map<String, Map<String, String>>> termIdPropMap = db .createHashMap("termIdPropMap") .make(); final Set<String> referencedSourceIds = db .createHashSet("sourceIdMap") .make(); final Set<String> referencedTargetIds = db .createHashSet("targetIdMap") .make(); for (Record record : resourceExtension) { String targetId = record.value(DwcTerm.relatedResourceID); String sourceId = record.value(DwcTerm.resourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { referencedSourceIds.add(sourceId); referencedTargetIds.add(targetId); } } final List<DwcTerm> idTerms = Arrays.asList( DwcTerm.occurrenceID, DwcTerm.taxonID); List<ArchiveFile> archiveFiles = new ArrayList<>(); archiveFiles.add(archive.getCore()); ArchiveFile taxon = findResourceExtension(archive, EXTENSION_TAXON); if (taxon != null) { archiveFiles.add(taxon); } for (ArchiveFile archiveFile : archiveFiles) { for (Record record : archiveFile) { for (DwcTerm idTerm : idTerms) { attemptLinkUsingTerm(termIdPropMap, referencedSourceIds, referencedTargetIds, record, idTerm); } } } for (Record record : resourceExtension) { Map<String, String> props = new TreeMap<>(); String sourceId = record.value(DwcTerm.resourceID); String relationship = record.value(DwcTerm.relationshipOfResource); Optional<Term> relationshipOfResourceIDTerm = record .terms() .stream() .filter(x -> StringUtils.equals(x.simpleName(), "relationshipOfResourceID")) .findFirst(); String relationshipTypeIdValue = relationshipOfResourceIDTerm .map(record::value) .orElse(null); String targetId = record.value(DwcTerm.relatedResourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { appendVerbatimResourceRelationsValues(record, props); String relationshipAccordingTo = record.value(DwcTerm.relationshipAccordingTo); if (StringUtils.isNotBlank(relationshipAccordingTo)) { props.putIfAbsent(REFERENCE_CITATION, relationshipAccordingTo); } putIfAbsentAndNotBlank(props, INTERACTION_TYPE_NAME, relationship); putIfAbsentAndNotBlank(props, INTERACTION_TYPE_ID, relationshipTypeIdValue); putIfAbsentAndNotBlank(props, DatasetImporterForMetaTable.EVENT_DATE, record.value(DwcTerm.relationshipEstablishedDate)); for (DwcTerm idTerm : idTerms) { if (termIdPropMap.containsKey(idTerm.qualifiedName())) { Map<String, Map<String, String>> propMap = termIdPropMap.get(idTerm.qualifiedName()); Map<String, String> sourceIdProperties = propMap.get(sourceId); Pair<String, String> idLabelPair = Pair.of( SOURCE_OCCURRENCE_ID, TARGET_OCCURRENCE_ID); if (idTerm.equals(DwcTerm.taxonID)) { idLabelPair = Pair.of(SOURCE_TAXON_ID, TARGET_TAXON_ID); } populatePropertiesAssociatedWithId(props, sourceId, true, sourceIdProperties, idLabelPair); Map<String, String> targetIdProperties = propMap.get(targetId); populatePropertiesAssociatedWithId(props, targetId, false, targetIdProperties, idLabelPair); } } try { interactionListener.newLink(props); } catch (StudyImporterException e) { } } } } } DatasetImporterForDwCA(ParserFactory parserFactory, NodeFactory nodeFactory); }
DatasetImporterForDwCA extends DatasetImporterWithListener { static void importResourceRelationExtension(Archive archive, InteractionListener interactionListener) { ArchiveFile resourceExtension = findResourceRelationshipExtension(archive); if (resourceExtension != null) { DB db = DBMaker .newMemoryDirectDB() .compressionEnable() .transactionDisable() .make(); final HTreeMap<String, Map<String, Map<String, String>>> termIdPropMap = db .createHashMap("termIdPropMap") .make(); final Set<String> referencedSourceIds = db .createHashSet("sourceIdMap") .make(); final Set<String> referencedTargetIds = db .createHashSet("targetIdMap") .make(); for (Record record : resourceExtension) { String targetId = record.value(DwcTerm.relatedResourceID); String sourceId = record.value(DwcTerm.resourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { referencedSourceIds.add(sourceId); referencedTargetIds.add(targetId); } } final List<DwcTerm> idTerms = Arrays.asList( DwcTerm.occurrenceID, DwcTerm.taxonID); List<ArchiveFile> archiveFiles = new ArrayList<>(); archiveFiles.add(archive.getCore()); ArchiveFile taxon = findResourceExtension(archive, EXTENSION_TAXON); if (taxon != null) { archiveFiles.add(taxon); } for (ArchiveFile archiveFile : archiveFiles) { for (Record record : archiveFile) { for (DwcTerm idTerm : idTerms) { attemptLinkUsingTerm(termIdPropMap, referencedSourceIds, referencedTargetIds, record, idTerm); } } } for (Record record : resourceExtension) { Map<String, String> props = new TreeMap<>(); String sourceId = record.value(DwcTerm.resourceID); String relationship = record.value(DwcTerm.relationshipOfResource); Optional<Term> relationshipOfResourceIDTerm = record .terms() .stream() .filter(x -> StringUtils.equals(x.simpleName(), "relationshipOfResourceID")) .findFirst(); String relationshipTypeIdValue = relationshipOfResourceIDTerm .map(record::value) .orElse(null); String targetId = record.value(DwcTerm.relatedResourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { appendVerbatimResourceRelationsValues(record, props); String relationshipAccordingTo = record.value(DwcTerm.relationshipAccordingTo); if (StringUtils.isNotBlank(relationshipAccordingTo)) { props.putIfAbsent(REFERENCE_CITATION, relationshipAccordingTo); } putIfAbsentAndNotBlank(props, INTERACTION_TYPE_NAME, relationship); putIfAbsentAndNotBlank(props, INTERACTION_TYPE_ID, relationshipTypeIdValue); putIfAbsentAndNotBlank(props, DatasetImporterForMetaTable.EVENT_DATE, record.value(DwcTerm.relationshipEstablishedDate)); for (DwcTerm idTerm : idTerms) { if (termIdPropMap.containsKey(idTerm.qualifiedName())) { Map<String, Map<String, String>> propMap = termIdPropMap.get(idTerm.qualifiedName()); Map<String, String> sourceIdProperties = propMap.get(sourceId); Pair<String, String> idLabelPair = Pair.of( SOURCE_OCCURRENCE_ID, TARGET_OCCURRENCE_ID); if (idTerm.equals(DwcTerm.taxonID)) { idLabelPair = Pair.of(SOURCE_TAXON_ID, TARGET_TAXON_ID); } populatePropertiesAssociatedWithId(props, sourceId, true, sourceIdProperties, idLabelPair); Map<String, String> targetIdProperties = propMap.get(targetId); populatePropertiesAssociatedWithId(props, targetId, false, targetIdProperties, idLabelPair); } } try { interactionListener.newLink(props); } catch (StudyImporterException e) { } } } } } DatasetImporterForDwCA(ParserFactory parserFactory, NodeFactory nodeFactory); @Override void importStudy(); int importCore(Archive archive, InteractionListener interactionListener); void handleRecord(InteractionListener interactionListener, Record rec); static void addRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static Map<String, String> parseRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(String occurrenceRemarks); void addUSNMStyleHostOccurrenceRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static void mapCoreProperties(Record rec, Map<String, String> interactionProperties); static void mapLocationAndReferenceInfo(Record rec, Map<String, String> interactionProperties); static void mapSourceProperties(Record rec, Map<String, String> interactionProperties); static void mapIfAvailable(Record rec, Map<String, String> interactionProperties, String key, Term term); static void mapIfAvailable(Map<String, String> interactionProperties, String key, String value); static Map<String, String> parseUSNMStyleHostOccurrenceRemarks(String occurrenceRemarks); static Map<String, String> parseJsonChunk(String candidateJsonChunk); }
DatasetImporterForDwCA extends DatasetImporterWithListener { static void importResourceRelationExtension(Archive archive, InteractionListener interactionListener) { ArchiveFile resourceExtension = findResourceRelationshipExtension(archive); if (resourceExtension != null) { DB db = DBMaker .newMemoryDirectDB() .compressionEnable() .transactionDisable() .make(); final HTreeMap<String, Map<String, Map<String, String>>> termIdPropMap = db .createHashMap("termIdPropMap") .make(); final Set<String> referencedSourceIds = db .createHashSet("sourceIdMap") .make(); final Set<String> referencedTargetIds = db .createHashSet("targetIdMap") .make(); for (Record record : resourceExtension) { String targetId = record.value(DwcTerm.relatedResourceID); String sourceId = record.value(DwcTerm.resourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { referencedSourceIds.add(sourceId); referencedTargetIds.add(targetId); } } final List<DwcTerm> idTerms = Arrays.asList( DwcTerm.occurrenceID, DwcTerm.taxonID); List<ArchiveFile> archiveFiles = new ArrayList<>(); archiveFiles.add(archive.getCore()); ArchiveFile taxon = findResourceExtension(archive, EXTENSION_TAXON); if (taxon != null) { archiveFiles.add(taxon); } for (ArchiveFile archiveFile : archiveFiles) { for (Record record : archiveFile) { for (DwcTerm idTerm : idTerms) { attemptLinkUsingTerm(termIdPropMap, referencedSourceIds, referencedTargetIds, record, idTerm); } } } for (Record record : resourceExtension) { Map<String, String> props = new TreeMap<>(); String sourceId = record.value(DwcTerm.resourceID); String relationship = record.value(DwcTerm.relationshipOfResource); Optional<Term> relationshipOfResourceIDTerm = record .terms() .stream() .filter(x -> StringUtils.equals(x.simpleName(), "relationshipOfResourceID")) .findFirst(); String relationshipTypeIdValue = relationshipOfResourceIDTerm .map(record::value) .orElse(null); String targetId = record.value(DwcTerm.relatedResourceID); if (StringUtils.isNotBlank(sourceId) && StringUtils.isNotBlank(targetId)) { appendVerbatimResourceRelationsValues(record, props); String relationshipAccordingTo = record.value(DwcTerm.relationshipAccordingTo); if (StringUtils.isNotBlank(relationshipAccordingTo)) { props.putIfAbsent(REFERENCE_CITATION, relationshipAccordingTo); } putIfAbsentAndNotBlank(props, INTERACTION_TYPE_NAME, relationship); putIfAbsentAndNotBlank(props, INTERACTION_TYPE_ID, relationshipTypeIdValue); putIfAbsentAndNotBlank(props, DatasetImporterForMetaTable.EVENT_DATE, record.value(DwcTerm.relationshipEstablishedDate)); for (DwcTerm idTerm : idTerms) { if (termIdPropMap.containsKey(idTerm.qualifiedName())) { Map<String, Map<String, String>> propMap = termIdPropMap.get(idTerm.qualifiedName()); Map<String, String> sourceIdProperties = propMap.get(sourceId); Pair<String, String> idLabelPair = Pair.of( SOURCE_OCCURRENCE_ID, TARGET_OCCURRENCE_ID); if (idTerm.equals(DwcTerm.taxonID)) { idLabelPair = Pair.of(SOURCE_TAXON_ID, TARGET_TAXON_ID); } populatePropertiesAssociatedWithId(props, sourceId, true, sourceIdProperties, idLabelPair); Map<String, String> targetIdProperties = propMap.get(targetId); populatePropertiesAssociatedWithId(props, targetId, false, targetIdProperties, idLabelPair); } } try { interactionListener.newLink(props); } catch (StudyImporterException e) { } } } } } DatasetImporterForDwCA(ParserFactory parserFactory, NodeFactory nodeFactory); @Override void importStudy(); int importCore(Archive archive, InteractionListener interactionListener); void handleRecord(InteractionListener interactionListener, Record rec); static void addRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static Map<String, String> parseRoyalSaskatchewanMuseumOwlPelletCollectionStyleRemarks(String occurrenceRemarks); void addUSNMStyleHostOccurrenceRemarks(List<Map<String, String>> interactionCandidates, String occurrenceRemarks); static void mapCoreProperties(Record rec, Map<String, String> interactionProperties); static void mapLocationAndReferenceInfo(Record rec, Map<String, String> interactionProperties); static void mapSourceProperties(Record rec, Map<String, String> interactionProperties); static void mapIfAvailable(Record rec, Map<String, String> interactionProperties, String key, Term term); static void mapIfAvailable(Map<String, String> interactionProperties, String key, String value); static Map<String, String> parseUSNMStyleHostOccurrenceRemarks(String occurrenceRemarks); static Map<String, String> parseJsonChunk(String candidateJsonChunk); static final String EXTENSION_ASSOCIATED_TAXA; static final String EXTENSION_RESOURCE_RELATIONSHIP; static final String EXTENSION_TAXON; static final Pattern EX_NOTATION; static final Pattern REARED_EX_NOTATION; static final Pattern PATTERN_ASSOCIATED_TAXA_IDEA; static final Pattern PATTERN_ASSOCIATED_TAXA_EAE; }