@Test public void testTokenBoundaries() { TurkishTokenizer t = TurkishTokenizer.ALL; List<Token> tokens = t.tokenize("bir av. geldi."); Token t0 = tokens.get(0); Assert.assertEquals("bir", t0.getText()); Assert.assertEquals(0, t0.getStart()); Assert.assertEquals(2, t0.getEnd()); Token t1 = tokens.get(1); Assert.assertEquals(" ", t1.getText()); Assert.assertEquals(3, t1.getStart()); Assert.assertEquals(3, t1.getEnd()); Token t2 = tokens.get(2); Assert.assertEquals("av.", t2.getText()); Assert.assertEquals(4, t2.getStart()); Assert.assertEquals(6, t2.getEnd()); Token t3 = tokens.get(3); Assert.assertEquals(" ", t3.getText()); Assert.assertEquals(7, t3.getStart()); Assert.assertEquals(7, t3.getEnd()); Token t4 = tokens.get(4); Assert.assertEquals("geldi", t4.getText()); Assert.assertEquals(8, t4.getStart()); Assert.assertEquals(12, t4.getEnd()); Token t5 = tokens.get(5); Assert.assertEquals(".", t5.getText()); Assert.assertEquals(13, t5.getStart()); Assert.assertEquals(13, t5.getEnd()); }
public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); static final TurkishTokenizer ALL; static final TurkishTokenizer DEFAULT; }