@Test public void testExtractFeatures() { TokenContainer tc = new TokenContainer(); tc.add("word", "W=word"); tc.add("123", "IsNumber"); tc.add("Uppercaseword", "W=Uppercaseword", "IsUpperCase"); tc.add("ALLUPPERCASEWORD", "W=ALLUPPERCASEWORD", "IsAllUpperCase"); tc.add(",", "W=,", "IsSeparator"); tc.add("@", "W=@", "IsNonAlphanum"); tc.add("Maluwang", "W=Maluwang", "IsUpperCase"); tc.add("Maluwang", "W=Maluwang", "IsUpperCase", "KeywordAddress"); tc.add("na", "W=na", "KeywordAddress"); tc.add("lansangan", "W=lansangan", "KeywordAddress"); tc.add(".", "W=.", "IsSeparator"); tc.add("les", "W=les"); tc.add("escaldes", "W=escaldes"); tc.add("les", "W=les", "KeywordCity"); tc.add("Escaldes", "W=Escaldes", "KeywordCity", "IsUpperCase"); tc.add("mhm", "W=mhm"); tc.add("U", "W=U", "KeywordCountry", "IsUpperCase", "IsAllUpperCase"); tc.add(".", "W=.", "IsSeparator", "KeywordCountry"); tc.add("S", "W=S", "KeywordCountry", "IsUpperCase", "IsAllUpperCase"); tc.add(".", "W=.", "IsSeparator", "KeywordCountry"); tc.add("A", "W=A", "KeywordCountry", "IsUpperCase", "IsAllUpperCase"); tc.add(".", "W=.", "IsSeparator", "KeywordCountry"); tc.add("New", "W=New", "IsUpperCase", "KeywordState"); tc.add("Hampshire", "W=Hampshire", "IsUpperCase", "KeywordState"); tc.add("KS", "W=KS", "IsAllUpperCase", "KeywordStateCode"); tc.add("du", "W=du", "KeywordStopWord"); DocumentAffiliation instance = new DocumentAffiliation(""); instance.setTokens(tc.tokens); EXTRACTOR.calculateFeatures(instance); tc.checkFeatures(); }
@Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } @SuppressWarnings("unchecked")  AffiliationFeatureExtractor();  AffiliationFeatureExtractor(List<String> commonWords);  AffiliationFeatureExtractor(List<BinaryTokenFeatureCalculator> binaryFeatures,
            List<KeywordFeatureCalculator<Token<AffiliationLabel>>> keywordFeatures,
            WordFeatureCalculator wordFeature); }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } @SuppressWarnings("unchecked")  AffiliationFeatureExtractor();  AffiliationFeatureExtractor(List<String> commonWords);  AffiliationFeatureExtractor(List<BinaryTokenFeatureCalculator> binaryFeatures,
            List<KeywordFeatureCalculator<Token<AffiliationLabel>>> keywordFeatures,
            WordFeatureCalculator wordFeature); @Override void calculateFeatures(DocumentAffiliation affiliation); }
AffiliationFeatureExtractor implements FeatureExtractor<DocumentAffiliation> { @Override public void calculateFeatures(DocumentAffiliation affiliation) { List<Token<AffiliationLabel>> tokens = affiliation.getTokens(); for (Token<AffiliationLabel> token : tokens) { for (BinaryTokenFeatureCalculator binaryFeatureCalculator : binaryFeatures) { if (binaryFeatureCalculator.calculateFeaturePredicate(token, affiliation)) { token.addFeature(binaryFeatureCalculator.getFeatureName()); } } String wordFeatureString = wordFeature.calculateFeatureValue(token, affiliation); if (wordFeatureString != null) { token.addFeature(wordFeatureString); } } for (KeywordFeatureCalculator<Token<AffiliationLabel>> dictionaryFeatureCalculator : keywordFeatures) { dictionaryFeatureCalculator.calculateDictionaryFeatures(tokens); } } @SuppressWarnings("unchecked")  AffiliationFeatureExtractor();  AffiliationFeatureExtractor(List<String> commonWords);  AffiliationFeatureExtractor(List<BinaryTokenFeatureCalculator> binaryFeatures,
            List<KeywordFeatureCalculator<Token<AffiliationLabel>>> keywordFeatures,
            WordFeatureCalculator wordFeature); @Override void calculateFeatures(DocumentAffiliation affiliation);  }