Tokenizers { public static Tokenizer transform(Tokenizer tokenizer, Function<String, String> function) { if (tokenizer instanceof Transform) { return Transform.createCombined( (Transform) tokenizer, function); } else if (tokenizer instanceof Filter) { return Transform.createCombined( (Filter) tokenizer, function); } return new Transform(tokenizer, function); } private  Tokenizers(); static Tokenizer pattern(Pattern pattern); static Tokenizer pattern(String regex); static Tokenizer qGram(int q); static Tokenizer qGramWithFilter(int q); static Tokenizer qGramWithPadding(int q); static Tokenizer qGramWithPadding(int q, String padding); static Tokenizer qGramWithPadding(int q, String startPadding,
			String endPadding); static Tokenizer whitespace(); static Tokenizer transform(Tokenizer tokenizer,
			Function<String, String> function); static Tokenizer chain(List<Tokenizer> tokenizers); static Tokenizer chain(Tokenizer tokenizer, Tokenizer... tokenizers); static Tokenizer filter(Tokenizer tokenizer,
			Predicate<String> predicate);  }
@Test public void shouldReturnTransform() { Tokenizer tokenizer = Tokenizers.transform(whitespace, identity); assertEquals(Transform.class, tokenizer.getClass()); Transform transform = (Transform) tokenizer; assertSame(whitespace, transform.getTokenizer()); assertSame(identity, transform.getFunction()); }
@Test public void shouldReturnTransformForTransform() { Transform t = new Transform(whitespace, identity); Tokenizer tokenizer = Tokenizers.transform(t, identity2); assertEquals(Transform.class, tokenizer.getClass()); Transform transform = (Transform) tokenizer; assertSame(whitespace, transform.getTokenizer()); assertEquals(Functions.compose(identity2, identity), transform.getFunction()); }
@Test public void shouldReturnTransformForFilter() { Tokenizers.Filter filter = new Tokenizers.Filter(whitespace, alwaysTrue); Tokenizer tokenizer = Tokenizers.transform(filter, identity); assertEquals(FilterTransform.class, tokenizer.getClass()); FilterTransform transform = (FilterTransform) tokenizer; assertSame(filter, transform.getTokenizer()); assertSame(identity, transform.getFunction()); }