@Test public void testStopwordsAreRemoved() { String stop = "stop"; String word = "word"; String white = "white"; String list = "list"; TokenSequence data = new TokenSequence( ImmutableList.of(new Token(stop), new Token(word), new Token(white), new Token(list))); Instance instance = new Instance(data, null, null, null); RemoveStopwords stopwords = new RemoveStopwords(ImmutableList.of(stop, word)); Instance output = stopwords.pipe(instance); TokenSequence ts = (TokenSequence) output.getData(); assertEquals(2, ts.size()); assertEquals( ImmutableSet.of(white, list), ts.stream().map(Token::getText).collect(Collectors.toSet())); }
@Override public Instance pipe(Instance carrier) { TokenSequence input = (TokenSequence) carrier.getData(); TokenSequence output = new TokenSequence(); for (int i = 0; i < input.size(); i++) { Token t = input.get(i); if (!stopwords.contains(t.getText())) { output.add(t); } } carrier.setData(output); return carrier; }
RemoveStopwords extends Pipe implements Serializable { @Override public Instance pipe(Instance carrier) { TokenSequence input = (TokenSequence) carrier.getData(); TokenSequence output = new TokenSequence(); for (int i = 0; i < input.size(); i++) { Token t = input.get(i); if (!stopwords.contains(t.getText())) { output.add(t); } } carrier.setData(output); return carrier; } }
RemoveStopwords extends Pipe implements Serializable { @Override public Instance pipe(Instance carrier) { TokenSequence input = (TokenSequence) carrier.getData(); TokenSequence output = new TokenSequence(); for (int i = 0; i < input.size(); i++) { Token t = input.get(i); if (!stopwords.contains(t.getText())) { output.add(t); } } carrier.setData(output); return carrier; } RemoveStopwords(Collection<String> stopwords); }
RemoveStopwords extends Pipe implements Serializable { @Override public Instance pipe(Instance carrier) { TokenSequence input = (TokenSequence) carrier.getData(); TokenSequence output = new TokenSequence(); for (int i = 0; i < input.size(); i++) { Token t = input.get(i); if (!stopwords.contains(t.getText())) { output.add(t); } } carrier.setData(output); return carrier; } RemoveStopwords(Collection<String> stopwords); @Override Instance pipe(Instance carrier); }
RemoveStopwords extends Pipe implements Serializable { @Override public Instance pipe(Instance carrier) { TokenSequence input = (TokenSequence) carrier.getData(); TokenSequence output = new TokenSequence(); for (int i = 0; i < input.size(); i++) { Token t = input.get(i); if (!stopwords.contains(t.getText())) { output.add(t); } } carrier.setData(output); return carrier; } RemoveStopwords(Collection<String> stopwords); @Override Instance pipe(Instance carrier);  }