TextUtil { static public int termFrequency(TextIndex textIndex, List<String> textTokens, List<String> termTokens){ if(textTokens.isEmpty() || termTokens.isEmpty()){ return 0; } boolean caseSensitive = textIndex.isCaseSensitive(); int maxLevenshteinDistance = textIndex.getMaxLevenshteinDistance(); if(maxLevenshteinDistance < 0){ throw new InvalidAttributeException(textIndex, PMMLAttributes.TEXTINDEX_MAXLEVENSHTEINDISTANCE, maxLevenshteinDistance); } boolean bestHits; TextIndex.CountHits countHits = textIndex.getCountHits(); switch(countHits){ case BEST_HITS: bestHits = true; break; case ALL_HITS: bestHits = false; break; default: throw new UnsupportedAttributeException(textIndex, countHits); } int maxFrequency; TextIndex.LocalTermWeights localTermWeights = textIndex.getLocalTermWeights(); switch(localTermWeights){ case BINARY: maxFrequency = 1; break; case TERM_FREQUENCY: case LOGARITHMIC: maxFrequency = Integer.MAX_VALUE; break; default: throw new UnsupportedAttributeException(textIndex, localTermWeights); } try { return termFrequency(textTokens, termTokens, caseSensitive, maxLevenshteinDistance, bestHits, maxFrequency); } catch(PMMLException pe){ throw pe.ensureContext(textIndex); } } private  TextUtil(); static String normalize(TextIndex textIndex, String string); static String normalize(TextIndex textIndex, TextIndexNormalization textIndexNormalization, String string); static List<String> tokenize(TextIndex textIndex, String text); static int termFrequency(TextIndex textIndex, List<String> textTokens, List<String> termTokens); static Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength); static boolean matches(TextIndex textIndex, List<String> leftTokens, List<String> rightTokens);  }
@Test public void termFrequency(){ List<String> textTokens = Arrays.asList("x", "x", "x", "x"); List<String> termTokens; assertEquals(0, termFrequency(textTokens, Arrays.asList("x", "x", "x", "x", "x"), true, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, Arrays.asList("x", "x", "x", "x"), true, 0, false, Integer.MAX_VALUE)); assertEquals(2, termFrequency(textTokens, Arrays.asList("x", "x", "x"), true, 0, false, Integer.MAX_VALUE)); assertEquals(3, termFrequency(textTokens, Arrays.asList("x", "x"), true, 0, false, Integer.MAX_VALUE)); assertEquals(4, termFrequency(textTokens, Arrays.asList("x"), true, 0, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "X", "x", "x"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 1, false, 1)); assertEquals(1, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "Y", "x", "x"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "X", "x"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(2, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 1, false, 1)); assertEquals(2, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(3, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 1, false, 1)); assertEquals(3, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(4, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 1, false, 1)); assertEquals(4, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "X", "X", "x"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 2, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "Y", "Y", "x"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 2, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "X", "X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(2, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); assertEquals(2, termFrequency(textTokens, termTokens, true, 2, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("X", "X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(3, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); assertEquals(3, termFrequency(textTokens, termTokens, true, 2, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "X", "X", "X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 2, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 3, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("x", "Y", "Y", "Y"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 3, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("X", "X", "X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(2, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 1, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, true, 2, false, Integer.MAX_VALUE)); assertEquals(2, termFrequency(textTokens, termTokens, true, 3, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("X", "X", "X", "X"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); termTokens = Arrays.asList("Y", "Y", "Y", "Y"); assertEquals(0, termFrequency(textTokens, termTokens, true, 0, false, Integer.MAX_VALUE)); assertEquals(0, termFrequency(textTokens, termTokens, false, 0, false, Integer.MAX_VALUE)); assertEquals(1, termFrequency(textTokens, termTokens, true, 4, false, Integer.MAX_VALUE)); }