@Test void createNGrams() { assertEquals( "{hy=1, te=1, s$=1, rt=1, pe=1, ns=1, yp=1, en=1, ^h=1, er=1}", NGramDistanceAlgorithm.createNGrams("hypertensions", true).toString()); assertEquals( "{d$=1, rt=1, or=1, ns=1, di=1, ^d=1, en=1, ^h=1, is=1, er=1, hy=1, te=1, s$=1, rd=1, pe=1, yp=1, so=1}", NGramDistanceAlgorithm.createNGrams("hypertensive disorder", true).toString()); assertEquals( "{d$=2, rt=1, or=2, ns=1, di=2, ^d=2, en=1, ^h=1, is=2, er=1, hy=1, te=1, s$=1, rd=2, pe=1, yp=1, so=2}", NGramDistanceAlgorithm.createNGrams("hypertensive disorder disorder", true).toString()); assertEquals( "{t$=1, pa=1, te=1, ^p=1, nt=1, ai=1, en=1, it=1}", NGramDistanceAlgorithm.createNGrams("WHERE IS PAitent", true).toString()); assertEquals( "{nt=1, ai=1, en=1, ^i=1, it=1, er=1, t$=1, wh=1, pa=1, te=1, ^p=1, re=1, ^w=1, i$=1, he=1, e$=1}", NGramDistanceAlgorithm.createNGrams("WHERE IS PAitent", false).toString()); }
public static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords) { List<String> wordsInString = Lists.newArrayList(Stemmer.replaceIllegalCharacter(inputQuery).split(" ")); if (removeStopWords) { wordsInString.removeAll(STOPWORDSLIST); } List<String> stemmedWordsInString = wordsInString.stream().map(Stemmer::stem).collect(Collectors.toList()); Map<String, Integer> tokens = new HashMap<>(); for (String singleWord : stemmedWordsInString) { if (!StringUtils.isEmpty(singleWord)) { StringBuilder singleString = new StringBuilder(singleWord.length() + 2); singleString.append('^').append(singleWord.toLowerCase()).append('$'); int length = singleString.length(); for (int i = 0; i < length - 1; i++) { String token = null; if (i + N_GRAMS < length) { token = singleString.substring(i, i + N_GRAMS); } else { token = singleString.substring(length - 2); } if (!tokens.containsKey(token)) { tokens.put(token, 1); } else { tokens.put(token, (tokens.get(token) + 1)); } } } } return tokens; }
NGramDistanceAlgorithm { public static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords) { List<String> wordsInString = Lists.newArrayList(Stemmer.replaceIllegalCharacter(inputQuery).split(" ")); if (removeStopWords) { wordsInString.removeAll(STOPWORDSLIST); } List<String> stemmedWordsInString = wordsInString.stream().map(Stemmer::stem).collect(Collectors.toList()); Map<String, Integer> tokens = new HashMap<>(); for (String singleWord : stemmedWordsInString) { if (!StringUtils.isEmpty(singleWord)) { StringBuilder singleString = new StringBuilder(singleWord.length() + 2); singleString.append('^').append(singleWord.toLowerCase()).append('$'); int length = singleString.length(); for (int i = 0; i < length - 1; i++) { String token = null; if (i + N_GRAMS < length) { token = singleString.substring(i, i + N_GRAMS); } else { token = singleString.substring(length - 2); } if (!tokens.containsKey(token)) { tokens.put(token, 1); } else { tokens.put(token, (tokens.get(token) + 1)); } } } } return tokens; } }
NGramDistanceAlgorithm { public static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords) { List<String> wordsInString = Lists.newArrayList(Stemmer.replaceIllegalCharacter(inputQuery).split(" ")); if (removeStopWords) { wordsInString.removeAll(STOPWORDSLIST); } List<String> stemmedWordsInString = wordsInString.stream().map(Stemmer::stem).collect(Collectors.toList()); Map<String, Integer> tokens = new HashMap<>(); for (String singleWord : stemmedWordsInString) { if (!StringUtils.isEmpty(singleWord)) { StringBuilder singleString = new StringBuilder(singleWord.length() + 2); singleString.append('^').append(singleWord.toLowerCase()).append('$'); int length = singleString.length(); for (int i = 0; i < length - 1; i++) { String token = null; if (i + N_GRAMS < length) { token = singleString.substring(i, i + N_GRAMS); } else { token = singleString.substring(length - 2); } if (!tokens.containsKey(token)) { tokens.put(token, 1); } else { tokens.put(token, (tokens.get(token) + 1)); } } } } return tokens; } private  NGramDistanceAlgorithm(); }
NGramDistanceAlgorithm { public static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords) { List<String> wordsInString = Lists.newArrayList(Stemmer.replaceIllegalCharacter(inputQuery).split(" ")); if (removeStopWords) { wordsInString.removeAll(STOPWORDSLIST); } List<String> stemmedWordsInString = wordsInString.stream().map(Stemmer::stem).collect(Collectors.toList()); Map<String, Integer> tokens = new HashMap<>(); for (String singleWord : stemmedWordsInString) { if (!StringUtils.isEmpty(singleWord)) { StringBuilder singleString = new StringBuilder(singleWord.length() + 2); singleString.append('^').append(singleWord.toLowerCase()).append('$'); int length = singleString.length(); for (int i = 0; i < length - 1; i++) { String token = null; if (i + N_GRAMS < length) { token = singleString.substring(i, i + N_GRAMS); } else { token = singleString.substring(length - 2); } if (!tokens.containsKey(token)) { tokens.put(token, 1); } else { tokens.put(token, (tokens.get(token) + 1)); } } } } return tokens; } private  NGramDistanceAlgorithm(); static double stringMatching(String queryOne, String queryTwo); static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords); }
NGramDistanceAlgorithm { public static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords) { List<String> wordsInString = Lists.newArrayList(Stemmer.replaceIllegalCharacter(inputQuery).split(" ")); if (removeStopWords) { wordsInString.removeAll(STOPWORDSLIST); } List<String> stemmedWordsInString = wordsInString.stream().map(Stemmer::stem).collect(Collectors.toList()); Map<String, Integer> tokens = new HashMap<>(); for (String singleWord : stemmedWordsInString) { if (!StringUtils.isEmpty(singleWord)) { StringBuilder singleString = new StringBuilder(singleWord.length() + 2); singleString.append('^').append(singleWord.toLowerCase()).append('$'); int length = singleString.length(); for (int i = 0; i < length - 1; i++) { String token = null; if (i + N_GRAMS < length) { token = singleString.substring(i, i + N_GRAMS); } else { token = singleString.substring(length - 2); } if (!tokens.containsKey(token)) { tokens.put(token, 1); } else { tokens.put(token, (tokens.get(token) + 1)); } } } } return tokens; } private  NGramDistanceAlgorithm(); static double stringMatching(String queryOne, String queryTwo); static Map<String, Integer> createNGrams(String inputQuery, boolean removeStopWords); static final Set<String> STOPWORDSLIST; }