@Test public void testUseAllTokens() { BagOfWordsFeatureGenerator generator = new BagOfWordsFeatureGenerator(); Assert.assertArrayEquals(new String[]{"bow=it", "bow=is", "bow=12.345", "bow=feet", "bow=long"}, generator.extractFeatures(new String[]{"it", "is", "12.345", "feet", "long"}, Collections.emptyMap()).toArray()); }
@Override public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) { Objects.requireNonNull(text, "text must not be null"); Collection<String> bagOfWords = new ArrayList<>(text.length); for (String word : text) { if (useOnlyAllLetterTokens) { StringPattern pattern = StringPattern.recognize(word); if (pattern.isAllLetter()) bagOfWords.add("bow=" + word); } else { bagOfWords.add("bow=" + word); } } return bagOfWords; }
BagOfWordsFeatureGenerator implements FeatureGenerator { @Override public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) { Objects.requireNonNull(text, "text must not be null"); Collection<String> bagOfWords = new ArrayList<>(text.length); for (String word : text) { if (useOnlyAllLetterTokens) { StringPattern pattern = StringPattern.recognize(word); if (pattern.isAllLetter()) bagOfWords.add("bow=" + word); } else { bagOfWords.add("bow=" + word); } } return bagOfWords; } }
BagOfWordsFeatureGenerator implements FeatureGenerator { @Override public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) { Objects.requireNonNull(text, "text must not be null"); Collection<String> bagOfWords = new ArrayList<>(text.length); for (String word : text) { if (useOnlyAllLetterTokens) { StringPattern pattern = StringPattern.recognize(word); if (pattern.isAllLetter()) bagOfWords.add("bow=" + word); } else { bagOfWords.add("bow=" + word); } } return bagOfWords; } BagOfWordsFeatureGenerator();   BagOfWordsFeatureGenerator(boolean useOnlyAllLetterTokens); }
BagOfWordsFeatureGenerator implements FeatureGenerator { @Override public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) { Objects.requireNonNull(text, "text must not be null"); Collection<String> bagOfWords = new ArrayList<>(text.length); for (String word : text) { if (useOnlyAllLetterTokens) { StringPattern pattern = StringPattern.recognize(word); if (pattern.isAllLetter()) bagOfWords.add("bow=" + word); } else { bagOfWords.add("bow=" + word); } } return bagOfWords; } BagOfWordsFeatureGenerator();   BagOfWordsFeatureGenerator(boolean useOnlyAllLetterTokens); @Override Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation); }
BagOfWordsFeatureGenerator implements FeatureGenerator { @Override public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) { Objects.requireNonNull(text, "text must not be null"); Collection<String> bagOfWords = new ArrayList<>(text.length); for (String word : text) { if (useOnlyAllLetterTokens) { StringPattern pattern = StringPattern.recognize(word); if (pattern.isAllLetter()) bagOfWords.add("bow=" + word); } else { bagOfWords.add("bow=" + word); } } return bagOfWords; } BagOfWordsFeatureGenerator();   BagOfWordsFeatureGenerator(boolean useOnlyAllLetterTokens); @Override Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation);  }