@Test public void testCalculateFeaturePredicate() { Token<AffiliationLabel> inSet = new Token<AffiliationLabel>("kot"); Token<AffiliationLabel> lowerInSet = new Token<AffiliationLabel>("KOT"); Token<AffiliationLabel> notInSet = new Token<AffiliationLabel>("SZCZUR"); Token<AffiliationLabel> notWord = new Token<AffiliationLabel>("kot1"); DocumentAffiliation aff = new DocumentAffiliation(""); assertEquals(false, FEATURE_CASE_SENSITIVE.calculateFeaturePredicate(inSet, aff)); assertEquals(true, FEATURE_CASE_SENSITIVE.calculateFeaturePredicate(lowerInSet, aff)); assertEquals(true, FEATURE_CASE_SENSITIVE.calculateFeaturePredicate(notInSet, aff)); assertEquals(false, FEATURE_CASE_SENSITIVE.calculateFeaturePredicate(notWord, aff)); assertEquals(false, FEATURE_IGNORE_CASE.calculateFeaturePredicate(inSet, aff)); assertEquals(false, FEATURE_IGNORE_CASE.calculateFeaturePredicate(lowerInSet, aff)); assertEquals(true, FEATURE_IGNORE_CASE.calculateFeaturePredicate(notInSet, aff)); assertEquals(false, FEATURE_IGNORE_CASE.calculateFeaturePredicate(notWord, aff)); }
@Override public boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context) { String text = token.getText(); if (!TextUtils.isWord(text)) { return false; } if (!caseSensitive) { text = text.toLowerCase(Locale.ENGLISH); } return !commonWords.contains(text); }
IsRareFeature extends BinaryTokenFeatureCalculator { @Override public boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context) { String text = token.getText(); if (!TextUtils.isWord(text)) { return false; } if (!caseSensitive) { text = text.toLowerCase(Locale.ENGLISH); } return !commonWords.contains(text); } }
IsRareFeature extends BinaryTokenFeatureCalculator { @Override public boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context) { String text = token.getText(); if (!TextUtils.isWord(text)) { return false; } if (!caseSensitive) { text = text.toLowerCase(Locale.ENGLISH); } return !commonWords.contains(text); } IsRareFeature(List<String> commonWordsList, boolean caseSensitive); }
IsRareFeature extends BinaryTokenFeatureCalculator { @Override public boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context) { String text = token.getText(); if (!TextUtils.isWord(text)) { return false; } if (!caseSensitive) { text = text.toLowerCase(Locale.ENGLISH); } return !commonWords.contains(text); } IsRareFeature(List<String> commonWordsList, boolean caseSensitive); @Override boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context); }
IsRareFeature extends BinaryTokenFeatureCalculator { @Override public boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context) { String text = token.getText(); if (!TextUtils.isWord(text)) { return false; } if (!caseSensitive) { text = text.toLowerCase(Locale.ENGLISH); } return !commonWords.contains(text); } IsRareFeature(List<String> commonWordsList, boolean caseSensitive); @Override boolean calculateFeaturePredicate(Token<?> token, ParsableString<?> context);  }