@Test public void testExtractFeaturesWithDocumentAffiliation() { String text = "Cóż ro123bić?"; List<List<String>> expectedFeatures = new ArrayList<List<String>>(); expectedFeatures.add(Arrays.asList("W=Coz", "IsUpperCase")); expectedFeatures.add(Arrays.asList("W=ro")); expectedFeatures.add(Arrays.asList("IsNumber")); expectedFeatures.add(Arrays.asList("W=bic")); expectedFeatures.add(Arrays.asList("W=?", "IsNonAlphanum")); DocumentAffiliation instance = new DocumentAffiliation(text); instance.setTokens(TOKENIZER.tokenize(instance.getRawText())); EXTRACTOR.calculateFeatures(instance); for (int i = 0; i < expectedFeatures.size(); i++) { List<String> expected = expectedFeatures.get(i); List<String> actual = instance.getTokens().get(i).getFeatures(); Collections.sort(expected); Collections.sort(actual); assertEquals(expected, actual); } }
@Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } @SuppressWarnings("unchecked")  AffiliationFeatureExtractor();  AffiliationFeatureExtractor(List<String> commonWords);  AffiliationFeatureExtractor(List<BinaryTokenFeatureCalculator> binaryFeatures,
            List<KeywordFeatureCalculator<Token<AffiliationLabel>>> keywordFeatures,
            WordFeatureCalculator wordFeature); }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } @SuppressWarnings("unchecked")  AffiliationFeatureExtractor();  AffiliationFeatureExtractor(List<String> commonWords);  AffiliationFeatureExtractor(List<BinaryTokenFeatureCalculator> binaryFeatures,
            List<KeywordFeatureCalculator<Token<AffiliationLabel>>> keywordFeatures,
            WordFeatureCalculator wordFeature); @Override void calculateFeatures(DocumentAffiliation affiliation); }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } @SuppressWarnings("unchecked")  AffiliationFeatureExtractor();  AffiliationFeatureExtractor(List<String> commonWords);  AffiliationFeatureExtractor(List<BinaryTokenFeatureCalculator> binaryFeatures,
            List<KeywordFeatureCalculator<Token<AffiliationLabel>>> keywordFeatures,
            WordFeatureCalculator wordFeature); @Override void calculateFeatures(DocumentAffiliation affiliation);  }