@Test public void termFrequencyTable(){ List<String> textTokens = Arrays.asList("a", "b", "A", "c"); Set<List<String>> termTokenSet = new HashSet<List<String>>(){ { add(Arrays.asList("a")); add(Arrays.asList("a", "b")); add(Arrays.asList("a", "b", "c")); } }; Map<List<String>, Integer> expectedTermFrequencyTable = new HashMap<>(); expectedTermFrequencyTable.put(Arrays.asList("a"), 2); Map<List<String>, Integer> termFrequencyTable = TextUtil.termFrequencyTable(textTokens, termTokenSet, false, 0, 1); assertEquals(expectedTermFrequencyTable, termFrequencyTable); expectedTermFrequencyTable.put(Arrays.asList("a", "b"), 1); termFrequencyTable = TextUtil.termFrequencyTable(textTokens, termTokenSet, false, 0, 2); assertEquals(expectedTermFrequencyTable, termFrequencyTable); expectedTermFrequencyTable = new HashMap<>(); expectedTermFrequencyTable.put(Arrays.asList("a"), 4); termFrequencyTable = TextUtil.termFrequencyTable(textTokens, termTokenSet, false, 1, 1); assertEquals(expectedTermFrequencyTable, termFrequencyTable); termFrequencyTable = TextUtil.termFrequencyTable(textTokens, termTokenSet, true, 1, 1); assertEquals(expectedTermFrequencyTable, termFrequencyTable); expectedTermFrequencyTable.put(Arrays.asList("a", "b"), 2); termFrequencyTable = TextUtil.termFrequencyTable(textTokens, termTokenSet, false, 1, 2); assertEquals(expectedTermFrequencyTable, termFrequencyTable); expectedTermFrequencyTable.put(Arrays.asList("a", "b"), 1); termFrequencyTable = TextUtil.termFrequencyTable(textTokens, termTokenSet, true, 1, 2); assertEquals(expectedTermFrequencyTable, termFrequencyTable); }
static public Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength){ boolean caseSensitive = textIndex.isCaseSensitive(); int maxLevenshteinDistance = textIndex.getMaxLevenshteinDistance(); if(maxLevenshteinDistance < 0){ throw new InvalidAttributeException(textIndex, PMMLAttributes.TEXTINDEX_MAXLEVENSHTEINDISTANCE, maxLevenshteinDistance); } return termFrequencyTable(textTokens, termTokenSet, caseSensitive, maxLevenshteinDistance, maxLength); }
TextUtil { static public Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength){ boolean caseSensitive = textIndex.isCaseSensitive(); int maxLevenshteinDistance = textIndex.getMaxLevenshteinDistance(); if(maxLevenshteinDistance < 0){ throw new InvalidAttributeException(textIndex, PMMLAttributes.TEXTINDEX_MAXLEVENSHTEINDISTANCE, maxLevenshteinDistance); } return termFrequencyTable(textTokens, termTokenSet, caseSensitive, maxLevenshteinDistance, maxLength); } }
TextUtil { static public Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength){ boolean caseSensitive = textIndex.isCaseSensitive(); int maxLevenshteinDistance = textIndex.getMaxLevenshteinDistance(); if(maxLevenshteinDistance < 0){ throw new InvalidAttributeException(textIndex, PMMLAttributes.TEXTINDEX_MAXLEVENSHTEINDISTANCE, maxLevenshteinDistance); } return termFrequencyTable(textTokens, termTokenSet, caseSensitive, maxLevenshteinDistance, maxLength); } private  TextUtil(); }
TextUtil { static public Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength){ boolean caseSensitive = textIndex.isCaseSensitive(); int maxLevenshteinDistance = textIndex.getMaxLevenshteinDistance(); if(maxLevenshteinDistance < 0){ throw new InvalidAttributeException(textIndex, PMMLAttributes.TEXTINDEX_MAXLEVENSHTEINDISTANCE, maxLevenshteinDistance); } return termFrequencyTable(textTokens, termTokenSet, caseSensitive, maxLevenshteinDistance, maxLength); } private  TextUtil(); static String normalize(TextIndex textIndex, String string); static String normalize(TextIndex textIndex, TextIndexNormalization textIndexNormalization, String string); static List<String> tokenize(TextIndex textIndex, String text); static int termFrequency(TextIndex textIndex, List<String> textTokens, List<String> termTokens); static Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength); static boolean matches(TextIndex textIndex, List<String> leftTokens, List<String> rightTokens); }
TextUtil { static public Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength){ boolean caseSensitive = textIndex.isCaseSensitive(); int maxLevenshteinDistance = textIndex.getMaxLevenshteinDistance(); if(maxLevenshteinDistance < 0){ throw new InvalidAttributeException(textIndex, PMMLAttributes.TEXTINDEX_MAXLEVENSHTEINDISTANCE, maxLevenshteinDistance); } return termFrequencyTable(textTokens, termTokenSet, caseSensitive, maxLevenshteinDistance, maxLength); } private  TextUtil(); static String normalize(TextIndex textIndex, String string); static String normalize(TextIndex textIndex, TextIndexNormalization textIndexNormalization, String string); static List<String> tokenize(TextIndex textIndex, String text); static int termFrequency(TextIndex textIndex, List<String> textTokens, List<String> termTokens); static Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength); static boolean matches(TextIndex textIndex, List<String> leftTokens, List<String> rightTokens);  }