@Test void testTokenize() { Set<String> tokenize = unitResolverImpl.tokenize("area density (kg/m^²)"); assertTrue(newHashSet("area", "density", "kg/m^²").containsAll(tokenize)); Set<String> tokenize1 = unitResolverImpl.tokenize("area density (kg/m^2)"); assertTrue(newHashSet("area", "density", "kg/m^²").containsAll(tokenize1)); Set<String> tokenize2 = unitResolverImpl.tokenize("area density (kg/m2)"); assertTrue(newHashSet("area", "density", "kg/m²").containsAll(tokenize2)); Set<String> tokenize3 = unitResolverImpl.tokenize("area 2 density2 (kg/m2)"); assertTrue(newHashSet("area", "density²", "kg/m²").containsAll(tokenize3)); Set<String> tokenize4 = unitResolverImpl.tokenize("area 2 density 2 (kg/m2)"); assertEquals(3, tokenize4.size()); assertFalse(tokenize4.containsAll(newHashSet("area", "density", "²", "kg/m²"))); }
Set<String> tokenize(String... terms) { Set<String> tokens = new HashSet<>(); if (terms != null && terms.length > 0) { Sets.newHashSet(terms).stream() .filter(StringUtils::isNotBlank) .map(StringUtils::lowerCase) .map(this::replaceIllegalChars) .forEach( term -> tokens.addAll( Sets.newHashSet(term.split("\\s+")).stream() .filter(this::notPureNumberExpression) .map(UnitHelper::numberToSuperscript) .collect(Collectors.toSet()))); tokens.removeAll(NGramDistanceAlgorithm.STOPWORDSLIST); } return tokens; }
UnitResolverImpl implements UnitResolver { Set<String> tokenize(String... terms) { Set<String> tokens = new HashSet<>(); if (terms != null && terms.length > 0) { Sets.newHashSet(terms).stream() .filter(StringUtils::isNotBlank) .map(StringUtils::lowerCase) .map(this::replaceIllegalChars) .forEach( term -> tokens.addAll( Sets.newHashSet(term.split("\\s+")).stream() .filter(this::notPureNumberExpression) .map(UnitHelper::numberToSuperscript) .collect(Collectors.toSet()))); tokens.removeAll(NGramDistanceAlgorithm.STOPWORDSLIST); } return tokens; } }
UnitResolverImpl implements UnitResolver { Set<String> tokenize(String... terms) { Set<String> tokens = new HashSet<>(); if (terms != null && terms.length > 0) { Sets.newHashSet(terms).stream() .filter(StringUtils::isNotBlank) .map(StringUtils::lowerCase) .map(this::replaceIllegalChars) .forEach( term -> tokens.addAll( Sets.newHashSet(term.split("\\s+")).stream() .filter(this::notPureNumberExpression) .map(UnitHelper::numberToSuperscript) .collect(Collectors.toSet()))); tokens.removeAll(NGramDistanceAlgorithm.STOPWORDSLIST); } return tokens; } UnitResolverImpl(OntologyService ontologyService); }
UnitResolverImpl implements UnitResolver { Set<String> tokenize(String... terms) { Set<String> tokens = new HashSet<>(); if (terms != null && terms.length > 0) { Sets.newHashSet(terms).stream() .filter(StringUtils::isNotBlank) .map(StringUtils::lowerCase) .map(this::replaceIllegalChars) .forEach( term -> tokens.addAll( Sets.newHashSet(term.split("\\s+")).stream() .filter(this::notPureNumberExpression) .map(UnitHelper::numberToSuperscript) .collect(Collectors.toSet()))); tokens.removeAll(NGramDistanceAlgorithm.STOPWORDSLIST); } return tokens; } UnitResolverImpl(OntologyService ontologyService); @Override Unit<? extends Quantity> resolveUnit(Attribute attr); }
UnitResolverImpl implements UnitResolver { Set<String> tokenize(String... terms) { Set<String> tokens = new HashSet<>(); if (terms != null && terms.length > 0) { Sets.newHashSet(terms).stream() .filter(StringUtils::isNotBlank) .map(StringUtils::lowerCase) .map(this::replaceIllegalChars) .forEach( term -> tokens.addAll( Sets.newHashSet(term.split("\\s+")).stream() .filter(this::notPureNumberExpression) .map(UnitHelper::numberToSuperscript) .collect(Collectors.toSet()))); tokens.removeAll(NGramDistanceAlgorithm.STOPWORDSLIST); } return tokens; } UnitResolverImpl(OntologyService ontologyService); @Override Unit<? extends Quantity> resolveUnit(Attribute attr);  }