@Test public void matchDataset() throws Exception { ClbConfiguration clb = new ClbConfiguration(); clb.serverName = "pg1.gbif.org"; clb.databaseName = "prod_checklistbank"; clb.user = "clb"; clb.password = "%BBJu2MgstXJ"; IdLookup lookup = IdLookupImpl.temp().load(clb, false); NubMatchService service = new NubMatchService(clb, lookup, null, null); service.matchDataset(UUID.fromString("38f91564-30ff-47b0-aa6b-cf3b59f0fca7")); }
public DatasetMatchSummary matchDataset(UUID key) throws DatasetMatchFailed { Dataset d = new Dataset(); d.setKey(key); d.setTitle("Dataset " + key); final DatasetMatchSummary summary = new DatasetMatchSummary(d.getKey()); if (Constants.NUB_DATASET_KEY.equals(d.getKey())) { LOG.warn("Cannot match backbone to itself. Ignore"); return summary; } LOG.info("Rematch checklist {} to Backbone", d.getKey()); Map<Integer, Integer> relations = Maps.newHashMap(); try (ClbSource src = new ClbSource(cfg, d)) { LOG.info("Copy usages for {} from pg into neo", d.getKey()); src.init(false, false); NubUsage unknown = new NubUsage(); unknown.usageKey = Kingdom.INCERTAE_SEDIS.nubUsageKey(); unknown.kingdom = Kingdom.INCERTAE_SEDIS; ParentStack parents = new ParentStack(unknown); for (SrcUsage u : src) { parents.add(u); if (u.parsedName.isParsableType() && !u.parsedName.isParsed()) { summary.addUnparsable(); } LookupUsage match; if (u.parsedName.isParsed()) { match = nubLookup.match(u.parsedName.canonicalName(), u.parsedName.getAuthorship(), u.parsedName.getYear(), u.rank, null, parents.nubKingdom()); } else { match = nubLookup.match(u.scientificName, u.rank, parents.nubKingdom()); } if (match != null) { summary.addMatch(u.rank); relations.put(u.key, match.getKey()); NubUsage nub = new NubUsage(); nub.kingdom = match.getKingdom(); parents.put(nub); } else { summary.addNoMatch(u.rank); LOG.debug("No match for {} in dataset {}. Parsed name: {}", u, d.getKey(), u.parsedName); relations.put(u.key, null); } } if (summary.percBackboneRelevantNoMatches() < 25 || (summary.percMatches() < 25 && summary.getTotalUsages() > 25)) { LOG.warn("Only {}% of all names and {}% of genera and below in dataset {} were matching", summary.percMatches(), summary.percBackboneRelevantNoMatches(), d.getKey()); } if (sqlService != null) { LOG.info("Updating {} nub relations with {} ({}%) matches and {} unparsable names from dataset {}", relations.size(), summary.getMatches(), summary.percMatches(), summary.getUnparsable(), d.getKey() ); sqlService.insertNubRelations(d.getKey(), relations); solrService.insertNubRelations(d.getKey(), relations); counter++; } else { LOG.warn("No sql service configured to persist the matches for dataset {}!", d.getKey()); } } catch (Exception e) { LOG.error("Failed to match checklist {} {}", d.getKey(), d.getTitle()); throw new DatasetMatchFailed(d.getKey(), e); } LOG.info("{}", summary); return summary; }
NubMatchService { public DatasetMatchSummary matchDataset(UUID key) throws DatasetMatchFailed { Dataset d = new Dataset(); d.setKey(key); d.setTitle("Dataset " + key); final DatasetMatchSummary summary = new DatasetMatchSummary(d.getKey()); if (Constants.NUB_DATASET_KEY.equals(d.getKey())) { LOG.warn("Cannot match backbone to itself. Ignore"); return summary; } LOG.info("Rematch checklist {} to Backbone", d.getKey()); Map<Integer, Integer> relations = Maps.newHashMap(); try (ClbSource src = new ClbSource(cfg, d)) { LOG.info("Copy usages for {} from pg into neo", d.getKey()); src.init(false, false); NubUsage unknown = new NubUsage(); unknown.usageKey = Kingdom.INCERTAE_SEDIS.nubUsageKey(); unknown.kingdom = Kingdom.INCERTAE_SEDIS; ParentStack parents = new ParentStack(unknown); for (SrcUsage u : src) { parents.add(u); if (u.parsedName.isParsableType() && !u.parsedName.isParsed()) { summary.addUnparsable(); } LookupUsage match; if (u.parsedName.isParsed()) { match = nubLookup.match(u.parsedName.canonicalName(), u.parsedName.getAuthorship(), u.parsedName.getYear(), u.rank, null, parents.nubKingdom()); } else { match = nubLookup.match(u.scientificName, u.rank, parents.nubKingdom()); } if (match != null) { summary.addMatch(u.rank); relations.put(u.key, match.getKey()); NubUsage nub = new NubUsage(); nub.kingdom = match.getKingdom(); parents.put(nub); } else { summary.addNoMatch(u.rank); LOG.debug("No match for {} in dataset {}. Parsed name: {}", u, d.getKey(), u.parsedName); relations.put(u.key, null); } } if (summary.percBackboneRelevantNoMatches() < 25 || (summary.percMatches() < 25 && summary.getTotalUsages() > 25)) { LOG.warn("Only {}% of all names and {}% of genera and below in dataset {} were matching", summary.percMatches(), summary.percBackboneRelevantNoMatches(), d.getKey()); } if (sqlService != null) { LOG.info("Updating {} nub relations with {} ({}%) matches and {} unparsable names from dataset {}", relations.size(), summary.getMatches(), summary.percMatches(), summary.getUnparsable(), d.getKey() ); sqlService.insertNubRelations(d.getKey(), relations); solrService.insertNubRelations(d.getKey(), relations); counter++; } else { LOG.warn("No sql service configured to persist the matches for dataset {}!", d.getKey()); } } catch (Exception e) { LOG.error("Failed to match checklist {} {}", d.getKey(), d.getTitle()); throw new DatasetMatchFailed(d.getKey(), e); } LOG.info("{}", summary); return summary; } }
NubMatchService { public DatasetMatchSummary matchDataset(UUID key) throws DatasetMatchFailed { Dataset d = new Dataset(); d.setKey(key); d.setTitle("Dataset " + key); final DatasetMatchSummary summary = new DatasetMatchSummary(d.getKey()); if (Constants.NUB_DATASET_KEY.equals(d.getKey())) { LOG.warn("Cannot match backbone to itself. Ignore"); return summary; } LOG.info("Rematch checklist {} to Backbone", d.getKey()); Map<Integer, Integer> relations = Maps.newHashMap(); try (ClbSource src = new ClbSource(cfg, d)) { LOG.info("Copy usages for {} from pg into neo", d.getKey()); src.init(false, false); NubUsage unknown = new NubUsage(); unknown.usageKey = Kingdom.INCERTAE_SEDIS.nubUsageKey(); unknown.kingdom = Kingdom.INCERTAE_SEDIS; ParentStack parents = new ParentStack(unknown); for (SrcUsage u : src) { parents.add(u); if (u.parsedName.isParsableType() && !u.parsedName.isParsed()) { summary.addUnparsable(); } LookupUsage match; if (u.parsedName.isParsed()) { match = nubLookup.match(u.parsedName.canonicalName(), u.parsedName.getAuthorship(), u.parsedName.getYear(), u.rank, null, parents.nubKingdom()); } else { match = nubLookup.match(u.scientificName, u.rank, parents.nubKingdom()); } if (match != null) { summary.addMatch(u.rank); relations.put(u.key, match.getKey()); NubUsage nub = new NubUsage(); nub.kingdom = match.getKingdom(); parents.put(nub); } else { summary.addNoMatch(u.rank); LOG.debug("No match for {} in dataset {}. Parsed name: {}", u, d.getKey(), u.parsedName); relations.put(u.key, null); } } if (summary.percBackboneRelevantNoMatches() < 25 || (summary.percMatches() < 25 && summary.getTotalUsages() > 25)) { LOG.warn("Only {}% of all names and {}% of genera and below in dataset {} were matching", summary.percMatches(), summary.percBackboneRelevantNoMatches(), d.getKey()); } if (sqlService != null) { LOG.info("Updating {} nub relations with {} ({}%) matches and {} unparsable names from dataset {}", relations.size(), summary.getMatches(), summary.percMatches(), summary.getUnparsable(), d.getKey() ); sqlService.insertNubRelations(d.getKey(), relations); solrService.insertNubRelations(d.getKey(), relations); counter++; } else { LOG.warn("No sql service configured to persist the matches for dataset {}!", d.getKey()); } } catch (Exception e) { LOG.error("Failed to match checklist {} {}", d.getKey(), d.getTitle()); throw new DatasetMatchFailed(d.getKey(), e); } LOG.info("{}", summary); return summary; } NubMatchService(ClbConfiguration cfg, IdLookup nubLookup, DatasetImportService sqlService, DatasetImportService solrService); }
NubMatchService { public DatasetMatchSummary matchDataset(UUID key) throws DatasetMatchFailed { Dataset d = new Dataset(); d.setKey(key); d.setTitle("Dataset " + key); final DatasetMatchSummary summary = new DatasetMatchSummary(d.getKey()); if (Constants.NUB_DATASET_KEY.equals(d.getKey())) { LOG.warn("Cannot match backbone to itself. Ignore"); return summary; } LOG.info("Rematch checklist {} to Backbone", d.getKey()); Map<Integer, Integer> relations = Maps.newHashMap(); try (ClbSource src = new ClbSource(cfg, d)) { LOG.info("Copy usages for {} from pg into neo", d.getKey()); src.init(false, false); NubUsage unknown = new NubUsage(); unknown.usageKey = Kingdom.INCERTAE_SEDIS.nubUsageKey(); unknown.kingdom = Kingdom.INCERTAE_SEDIS; ParentStack parents = new ParentStack(unknown); for (SrcUsage u : src) { parents.add(u); if (u.parsedName.isParsableType() && !u.parsedName.isParsed()) { summary.addUnparsable(); } LookupUsage match; if (u.parsedName.isParsed()) { match = nubLookup.match(u.parsedName.canonicalName(), u.parsedName.getAuthorship(), u.parsedName.getYear(), u.rank, null, parents.nubKingdom()); } else { match = nubLookup.match(u.scientificName, u.rank, parents.nubKingdom()); } if (match != null) { summary.addMatch(u.rank); relations.put(u.key, match.getKey()); NubUsage nub = new NubUsage(); nub.kingdom = match.getKingdom(); parents.put(nub); } else { summary.addNoMatch(u.rank); LOG.debug("No match for {} in dataset {}. Parsed name: {}", u, d.getKey(), u.parsedName); relations.put(u.key, null); } } if (summary.percBackboneRelevantNoMatches() < 25 || (summary.percMatches() < 25 && summary.getTotalUsages() > 25)) { LOG.warn("Only {}% of all names and {}% of genera and below in dataset {} were matching", summary.percMatches(), summary.percBackboneRelevantNoMatches(), d.getKey()); } if (sqlService != null) { LOG.info("Updating {} nub relations with {} ({}%) matches and {} unparsable names from dataset {}", relations.size(), summary.getMatches(), summary.percMatches(), summary.getUnparsable(), d.getKey() ); sqlService.insertNubRelations(d.getKey(), relations); solrService.insertNubRelations(d.getKey(), relations); counter++; } else { LOG.warn("No sql service configured to persist the matches for dataset {}!", d.getKey()); } } catch (Exception e) { LOG.error("Failed to match checklist {} {}", d.getKey(), d.getTitle()); throw new DatasetMatchFailed(d.getKey(), e); } LOG.info("{}", summary); return summary; } NubMatchService(ClbConfiguration cfg, IdLookup nubLookup, DatasetImportService sqlService, DatasetImportService solrService); int getCounter(); DatasetMatchSummary matchDataset(UUID key); }
NubMatchService { public DatasetMatchSummary matchDataset(UUID key) throws DatasetMatchFailed { Dataset d = new Dataset(); d.setKey(key); d.setTitle("Dataset " + key); final DatasetMatchSummary summary = new DatasetMatchSummary(d.getKey()); if (Constants.NUB_DATASET_KEY.equals(d.getKey())) { LOG.warn("Cannot match backbone to itself. Ignore"); return summary; } LOG.info("Rematch checklist {} to Backbone", d.getKey()); Map<Integer, Integer> relations = Maps.newHashMap(); try (ClbSource src = new ClbSource(cfg, d)) { LOG.info("Copy usages for {} from pg into neo", d.getKey()); src.init(false, false); NubUsage unknown = new NubUsage(); unknown.usageKey = Kingdom.INCERTAE_SEDIS.nubUsageKey(); unknown.kingdom = Kingdom.INCERTAE_SEDIS; ParentStack parents = new ParentStack(unknown); for (SrcUsage u : src) { parents.add(u); if (u.parsedName.isParsableType() && !u.parsedName.isParsed()) { summary.addUnparsable(); } LookupUsage match; if (u.parsedName.isParsed()) { match = nubLookup.match(u.parsedName.canonicalName(), u.parsedName.getAuthorship(), u.parsedName.getYear(), u.rank, null, parents.nubKingdom()); } else { match = nubLookup.match(u.scientificName, u.rank, parents.nubKingdom()); } if (match != null) { summary.addMatch(u.rank); relations.put(u.key, match.getKey()); NubUsage nub = new NubUsage(); nub.kingdom = match.getKingdom(); parents.put(nub); } else { summary.addNoMatch(u.rank); LOG.debug("No match for {} in dataset {}. Parsed name: {}", u, d.getKey(), u.parsedName); relations.put(u.key, null); } } if (summary.percBackboneRelevantNoMatches() < 25 || (summary.percMatches() < 25 && summary.getTotalUsages() > 25)) { LOG.warn("Only {}% of all names and {}% of genera and below in dataset {} were matching", summary.percMatches(), summary.percBackboneRelevantNoMatches(), d.getKey()); } if (sqlService != null) { LOG.info("Updating {} nub relations with {} ({}%) matches and {} unparsable names from dataset {}", relations.size(), summary.getMatches(), summary.percMatches(), summary.getUnparsable(), d.getKey() ); sqlService.insertNubRelations(d.getKey(), relations); solrService.insertNubRelations(d.getKey(), relations); counter++; } else { LOG.warn("No sql service configured to persist the matches for dataset {}!", d.getKey()); } } catch (Exception e) { LOG.error("Failed to match checklist {} {}", d.getKey(), d.getTitle()); throw new DatasetMatchFailed(d.getKey(), e); } LOG.info("{}", summary); return summary; } NubMatchService(ClbConfiguration cfg, IdLookup nubLookup, DatasetImportService sqlService, DatasetImportService solrService); int getCounter(); DatasetMatchSummary matchDataset(UUID key);  }