@Test public void evaluateTextIndex(){ FieldName name = FieldName.create("x"); TextIndex textIndex = new TextIndex(name, new Constant("user friendly")) .setWordSeparatorCharacterRE("[\\s\\-]"); assertEquals(null, evaluate(textIndex, name, null)); assertEquals(1, evaluate(textIndex, name, "user friendly")); assertEquals(1, evaluate(textIndex, name, "user-friendly")); textIndex = new TextIndex(name, new Constant("brown fox")); String text = "The quick browny foxy jumps over the lazy dog. The brown fox runs away and to be with another brown foxy."; textIndex.setMaxLevenshteinDistance(0); assertEquals(1, evaluate(textIndex, name, text)); textIndex.setMaxLevenshteinDistance(1); assertEquals(2, evaluate(textIndex, name, text)); textIndex.setMaxLevenshteinDistance(2); assertEquals(3, evaluate(textIndex, name, text)); textIndex = new TextIndex(name, new Constant("dog")) .setMaxLevenshteinDistance(1); text = "I have a doog. My dog is white. The doog is friendly."; textIndex.setCountHits(CountHits.ALL_HITS); assertEquals(3, evaluate(textIndex, name, text)); textIndex.setCountHits(CountHits.BEST_HITS); assertEquals(1, evaluate(textIndex, name, text)); textIndex = new TextIndex(name, new Constant("sun")) .setCaseSensitive(false); text = "The Sun was setting while the captain's son reached the bounty island, minutes after their ship had sunk to the bottom of the ocean."; textIndex.setMaxLevenshteinDistance(0); assertEquals(1, evaluate(textIndex, name, text)); textIndex.setMaxLevenshteinDistance(1); assertEquals(3, evaluate(textIndex, name, text)); }
static public FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context){ FieldName textName = textIndex.getTextField(); if(textName == null){ throw new MissingAttributeException(textIndex, PMMLAttributes.TEXTINDEX_TEXTFIELD); } FieldValue textValue = context.evaluate(textName); FieldValue termValue = ExpressionUtil.evaluateExpressionContainer(textIndex, context); if(FieldValueUtil.isMissing(textValue) || FieldValueUtil.isMissing(termValue)){ return FieldValues.MISSING_VALUE; } TextUtil.TextProcessor textProcessor = new TextUtil.TextProcessor(textIndex, textValue); List<String> textTokens = textProcessor.process(); TextUtil.TermProcessor termProcessor = new TextUtil.TermProcessor(textIndex, termValue); List<String> termTokens = termProcessor.process(); int termFrequency = TextUtil.termFrequency(textIndex, textTokens, termTokens); TextIndex.LocalTermWeights localTermWeights = textIndex.getLocalTermWeights(); switch(localTermWeights){ case BINARY: case TERM_FREQUENCY: return FieldValueUtil.create(TypeInfos.CONTINUOUS_INTEGER, termFrequency); case LOGARITHMIC: return FieldValueUtil.create(TypeInfos.CONTINUOUS_DOUBLE, Math.log10(1d + termFrequency)); default: throw new UnsupportedAttributeException(textIndex, localTermWeights); } }
ExpressionUtil { static public FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context){ FieldName textName = textIndex.getTextField(); if(textName == null){ throw new MissingAttributeException(textIndex, PMMLAttributes.TEXTINDEX_TEXTFIELD); } FieldValue textValue = context.evaluate(textName); FieldValue termValue = ExpressionUtil.evaluateExpressionContainer(textIndex, context); if(FieldValueUtil.isMissing(textValue) || FieldValueUtil.isMissing(termValue)){ return FieldValues.MISSING_VALUE; } TextUtil.TextProcessor textProcessor = new TextUtil.TextProcessor(textIndex, textValue); List<String> textTokens = textProcessor.process(); TextUtil.TermProcessor termProcessor = new TextUtil.TermProcessor(textIndex, termValue); List<String> termTokens = termProcessor.process(); int termFrequency = TextUtil.termFrequency(textIndex, textTokens, termTokens); TextIndex.LocalTermWeights localTermWeights = textIndex.getLocalTermWeights(); switch(localTermWeights){ case BINARY: case TERM_FREQUENCY: return FieldValueUtil.create(TypeInfos.CONTINUOUS_INTEGER, termFrequency); case LOGARITHMIC: return FieldValueUtil.create(TypeInfos.CONTINUOUS_DOUBLE, Math.log10(1d + termFrequency)); default: throw new UnsupportedAttributeException(textIndex, localTermWeights); } } }
ExpressionUtil { static public FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context){ FieldName textName = textIndex.getTextField(); if(textName == null){ throw new MissingAttributeException(textIndex, PMMLAttributes.TEXTINDEX_TEXTFIELD); } FieldValue textValue = context.evaluate(textName); FieldValue termValue = ExpressionUtil.evaluateExpressionContainer(textIndex, context); if(FieldValueUtil.isMissing(textValue) || FieldValueUtil.isMissing(termValue)){ return FieldValues.MISSING_VALUE; } TextUtil.TextProcessor textProcessor = new TextUtil.TextProcessor(textIndex, textValue); List<String> textTokens = textProcessor.process(); TextUtil.TermProcessor termProcessor = new TextUtil.TermProcessor(textIndex, termValue); List<String> termTokens = termProcessor.process(); int termFrequency = TextUtil.termFrequency(textIndex, textTokens, termTokens); TextIndex.LocalTermWeights localTermWeights = textIndex.getLocalTermWeights(); switch(localTermWeights){ case BINARY: case TERM_FREQUENCY: return FieldValueUtil.create(TypeInfos.CONTINUOUS_INTEGER, termFrequency); case LOGARITHMIC: return FieldValueUtil.create(TypeInfos.CONTINUOUS_DOUBLE, Math.log10(1d + termFrequency)); default: throw new UnsupportedAttributeException(textIndex, localTermWeights); } } private  ExpressionUtil(); }
ExpressionUtil { static public FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context){ FieldName textName = textIndex.getTextField(); if(textName == null){ throw new MissingAttributeException(textIndex, PMMLAttributes.TEXTINDEX_TEXTFIELD); } FieldValue textValue = context.evaluate(textName); FieldValue termValue = ExpressionUtil.evaluateExpressionContainer(textIndex, context); if(FieldValueUtil.isMissing(textValue) || FieldValueUtil.isMissing(termValue)){ return FieldValues.MISSING_VALUE; } TextUtil.TextProcessor textProcessor = new TextUtil.TextProcessor(textIndex, textValue); List<String> textTokens = textProcessor.process(); TextUtil.TermProcessor termProcessor = new TextUtil.TermProcessor(textIndex, termValue); List<String> termTokens = termProcessor.process(); int termFrequency = TextUtil.termFrequency(textIndex, textTokens, termTokens); TextIndex.LocalTermWeights localTermWeights = textIndex.getLocalTermWeights(); switch(localTermWeights){ case BINARY: case TERM_FREQUENCY: return FieldValueUtil.create(TypeInfos.CONTINUOUS_INTEGER, termFrequency); case LOGARITHMIC: return FieldValueUtil.create(TypeInfos.CONTINUOUS_DOUBLE, Math.log10(1d + termFrequency)); default: throw new UnsupportedAttributeException(textIndex, localTermWeights); } } private  ExpressionUtil(); static FieldName ensureField(E hasField); static Expression ensureExpression(E hasExpression); static FieldValue evaluateExpressionContainer(E hasExpression, EvaluationContext context); static FieldValue evaluateTypedExpressionContainer(E hasTypedExpression, EvaluationContext context); static FieldValue evaluate(DerivedField derivedField, EvaluationContext context); static FieldValue evaluate(DefineFunction defineFunction, List<FieldValue> values, EvaluationContext context); static FieldValue evaluate(Expression expression, EvaluationContext context); static FieldValue evaluateConstant(Constant constant); static FieldValue evaluateFieldRef(FieldRef fieldRef, EvaluationContext context); static FieldValue evaluateNormContinuous(NormContinuous normContinuous, EvaluationContext context); static FieldValue evaluateNormDiscrete(NormDiscrete normDiscrete, EvaluationContext context); static FieldValue evaluateDiscretize(Discretize discretize, EvaluationContext context); static FieldValue evaluateMapValues(MapValues mapValues, EvaluationContext context); static FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context); static FieldValue evaluateApply(Apply apply, EvaluationContext context); @SuppressWarnings ( value = {"unchecked"} ) static FieldValue evaluateAggregate(Aggregate aggregate, EvaluationContext context); static FieldValue evaluateJavaExpression(JavaExpression javaExpression, EvaluationContext context); static boolean isEmptyContent(Object value); }
ExpressionUtil { static public FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context){ FieldName textName = textIndex.getTextField(); if(textName == null){ throw new MissingAttributeException(textIndex, PMMLAttributes.TEXTINDEX_TEXTFIELD); } FieldValue textValue = context.evaluate(textName); FieldValue termValue = ExpressionUtil.evaluateExpressionContainer(textIndex, context); if(FieldValueUtil.isMissing(textValue) || FieldValueUtil.isMissing(termValue)){ return FieldValues.MISSING_VALUE; } TextUtil.TextProcessor textProcessor = new TextUtil.TextProcessor(textIndex, textValue); List<String> textTokens = textProcessor.process(); TextUtil.TermProcessor termProcessor = new TextUtil.TermProcessor(textIndex, termValue); List<String> termTokens = termProcessor.process(); int termFrequency = TextUtil.termFrequency(textIndex, textTokens, termTokens); TextIndex.LocalTermWeights localTermWeights = textIndex.getLocalTermWeights(); switch(localTermWeights){ case BINARY: case TERM_FREQUENCY: return FieldValueUtil.create(TypeInfos.CONTINUOUS_INTEGER, termFrequency); case LOGARITHMIC: return FieldValueUtil.create(TypeInfos.CONTINUOUS_DOUBLE, Math.log10(1d + termFrequency)); default: throw new UnsupportedAttributeException(textIndex, localTermWeights); } } private  ExpressionUtil(); static FieldName ensureField(E hasField); static Expression ensureExpression(E hasExpression); static FieldValue evaluateExpressionContainer(E hasExpression, EvaluationContext context); static FieldValue evaluateTypedExpressionContainer(E hasTypedExpression, EvaluationContext context); static FieldValue evaluate(DerivedField derivedField, EvaluationContext context); static FieldValue evaluate(DefineFunction defineFunction, List<FieldValue> values, EvaluationContext context); static FieldValue evaluate(Expression expression, EvaluationContext context); static FieldValue evaluateConstant(Constant constant); static FieldValue evaluateFieldRef(FieldRef fieldRef, EvaluationContext context); static FieldValue evaluateNormContinuous(NormContinuous normContinuous, EvaluationContext context); static FieldValue evaluateNormDiscrete(NormDiscrete normDiscrete, EvaluationContext context); static FieldValue evaluateDiscretize(Discretize discretize, EvaluationContext context); static FieldValue evaluateMapValues(MapValues mapValues, EvaluationContext context); static FieldValue evaluateTextIndex(TextIndex textIndex, EvaluationContext context); static FieldValue evaluateApply(Apply apply, EvaluationContext context); @SuppressWarnings ( value = {"unchecked"} ) static FieldValue evaluateAggregate(Aggregate aggregate, EvaluationContext context); static FieldValue evaluateJavaExpression(JavaExpression javaExpression, EvaluationContext context); static boolean isEmptyContent(Object value);  }