LtSegProvider extends AbstractStringProvider implements StringProviderMXBean { @Override public List<String>[] getNgrams(String text, String language_code) throws Exception { LOG.trace(String.format("Computing ngrams from text: %s", StringUtils.abbreviate(text, 200))); List<String>[] ngrams = null; for(String sentence : splitSentences(text, language_code)){ List<String> tokens = tokenizeSentence(sentence, language_code); if(tokens.isEmpty()) continue; LOG.trace(String.format("Current sentence: %s", StringUtils.abbreviate(tokens.toString(), 200))); List<String>[] current_ngrams = getNgramSequenceFromSentence(tokens); LOG.trace(String.format("Current ngrams: %s", StringUtils.abbreviate(Arrays.toString(current_ngrams), 200))); if (ngrams == null) ngrams = current_ngrams; else ngrams = ArrayUtils.getConcatinatedArray(ngrams, current_ngrams); } if (ngrams == null) ngrams = EMPTY_NGRAM_LIST; LOG.trace(String.format("Ngrams for text: '%s': %s ", StringUtils.abbreviate(text, 200), StringUtils.abbreviate(Arrays.toString(ngrams), 200))); return ngrams; } LtSegProvider(); @Override List<String>[] getNgrams(String text, String language_code); @Override List<String> splitSentences(String text, String language_code); @Override List<String> tokenizeSentence_intern(String sentence, String language_code);  }
@Test public void test() throws Exception { StringProviderMXBean strprovider = AbstractStringProvider.connectToServer(_host, _rmiport, _lm_id); System.out.println(strprovider.getPerplexity("Schöne neue Wörld. Schöne neue Wörld.", false)); System.out.println(Arrays.toString(strprovider.getNgrams("Schöne neue Wörld. Sch0ne neue W0r1d. !plan3t is now live on DuckDuckGo!"))); new DiffTokenizer().init(new StringReader("!plan3t is now live on DuckDuckGo!")).filteredAndNormalizedTokens(5, 2, true, false).forEach(System.out::println);; System.out.println(Arrays.toString(strprovider.getNgrams("M"))); }
@Test public void testFiltering() throws Exception { StringProviderMXBean strprovider = AbstractStringProvider.connectToServer(_host, _rmiport, _lm_id); List<String>[] ngrams = strprovider.getNgrams("The qu1ck br0wn fox jumps over the lazy dog."); System.out.println(Arrays.asList(ngrams)); }