@Test public void testInstances() { TurkishTokenizer t = TurkishTokenizer.DEFAULT; matchToken(t, "a b \t c \n \r", "a", "b", "c"); t = TurkishTokenizer.ALL; matchToken(t, " a b\t\n\rc", " ", "a", " ", "b", "\t", "\n", "\r", "c"); t = TurkishTokenizer.builder().ignoreAll().acceptTypes(Type.Number).build(); matchToken(t, "www.foo.bar 12,4'ü a@a.v ; ^% 2 adf 12 \r \n ", "12,4'ü", "2", "12"); }
public static Builder builder() { return new Builder(); }
TurkishTokenizer { public static Builder builder() { return new Builder(); } }
TurkishTokenizer { public static Builder builder() { return new Builder(); } private  TurkishTokenizer(long acceptedTypeBits); }
TurkishTokenizer { public static Builder builder() { return new Builder(); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); }
TurkishTokenizer { public static Builder builder() { return new Builder(); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); static final TurkishTokenizer ALL; static final TurkishTokenizer DEFAULT; }