NewlineTokenizer extends Tokenizer { @Override public void tokenize(Document input) { input.terms = Arrays.asList(input.text.split("\n")); input.tags = Collections.emptyList(); } NewlineTokenizer(TupleFlowParameters tfp);  NewlineTokenizer(); @Override void tokenize(Document input);  }
@Test public void testSimple() { NewlineTokenizer tok = new NewlineTokenizer(); Document ridiculous = tok.tokenize("this\nIs\tnot\na\nCAPitaliZATION\nunder_scored\n14.95.dots\ntest"); String[] expected = {"this", "Is\tnot", "a", "CAPitaliZATION", "under_scored", "14.95.dots", "test"}; assertEquals(ridiculous.terms.size(), expected.length); for(int i=0; i<expected.length; i++) { assertEquals(expected[i], ridiculous.terms.get(i)); } }