@Test @Ignore("Not an actual test. Requires external data.") public void performance() throws IOException { TurkishTokenizer tokenizer = TurkishTokenizer.DEFAULT; for (int it = 0; it < 5; it++) { List<String> lines = Files.readAllLines( Paths.get("/media/aaa/Data/aaa/corpora/dunya.100k")); Stopwatch clock = Stopwatch.createStarted(); long tokenCount = 0; for (String line : lines) { List<Token> tokens = tokenizer.tokenize(line); tokenCount += tokens.size(); } long elapsed = clock.elapsed(TimeUnit.MILLISECONDS); Log.info("Token count = %d ", tokenCount); Log.info("Speed (tps) = %.1f", tokenCount * 1000d / elapsed); } }
public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); }
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); static final TurkishTokenizer ALL; static final TurkishTokenizer DEFAULT; }