BreakIteratorStringProvider extends AbstractStringProvider implements StringProviderMXBean { @Override public List<String>[] getNgrams(String text, String language_code) throws Exception { LOG.trace(String.format("Computing ngrams from text: %s", StringUtils.abbreviate(text, 200))); List<String>[] ngrams = null; for(String sentence : splitSentences(text, language_code)){ List<String> tokens = tokenizeSentence(sentence, language_code); if(tokens.isEmpty()) continue; LOG.trace(String.format("Current sentence: %s", StringUtils.abbreviate(tokens.toString(), 200))); List<String>[] current_ngrams = getNgramSequenceFromSentence(tokens); LOG.trace(String.format("Current ngrams: %s", StringUtils.abbreviate(Arrays.toString(current_ngrams), 200))); if (ngrams == null) ngrams = current_ngrams; else ngrams = ArrayUtils.getConcatinatedArray(ngrams, current_ngrams); } if (ngrams == null) ngrams = EMPTY_NGRAM_LIST; LOG.trace(String.format("Ngrams for text: '%s': %s ", StringUtils.abbreviate(text, 200), StringUtils.abbreviate(Arrays.toString(ngrams), 200))); return ngrams; }  @Override List<String>[] getNgrams(String text, String language_code); @Override List<String> splitSentences(String text, String language_code); @Override List<String> tokenizeSentence_intern(String sentence, String language_code);  }
@Test public void test() throws Exception { StringProviderMXBean strprovider = AbstractStringProvider.connectToServer(_host, _rmiport, _lm_id); System.out.println(strprovider.getPerplexity("Schöne neue Wörld.", false)); System.out.println(Arrays.toString(strprovider.getNgrams("Schöne neue Wörld."))); System.out.println(Arrays.toString(strprovider.getNgrams("M"))); }
@Test public void testFiltering() throws Exception { StringProviderMXBean strprovider = AbstractStringProvider.connectToServer(_host, _rmiport, _lm_id); List<String>[] ngrams = strprovider.getNgrams("The qu1ck br0wn fox jumps over the lazy dog."); System.out.println(Arrays.asList(ngrams)); }
@Test public void testSentenceTagInsertion() throws Exception { String seq = "The qu1ck br0wn fox jumps over the lazy dog"; StringProviderMXBean provider = AbstractStringProvider.connectToServer(_host, _rmiport, _lm_id); String orig = Properties.get().getProperty("lt.lm.insertSentenceTags"); Properties.get().setProperty("lt.lm.insertSentenceTags", "0"); List<String>[] ngrams = provider.getNgrams(seq); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals("The", ngrams[0].get(0)); Assert.assertEquals("dog", ngrams[ngrams.length-1].get(provider.getLmOrder()-1)); Properties.get().setProperty("lt.lm.insertSentenceTags", "1"); ngrams = provider.getNgrams(seq); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals(PseudoSymbol.SEQUENCE_START.asString(), ngrams[0].get(0)); Assert.assertEquals("dog", ngrams[ngrams.length-1].get(provider.getLmOrder()-1)); Properties.get().setProperty("lt.lm.insertSentenceTags", "2"); ngrams = provider.getNgrams(seq); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals("The", ngrams[0].get(0)); Assert.assertEquals(PseudoSymbol.SEQUENCE_END.asString(), ngrams[ngrams.length-1].get(provider.getLmOrder()-1)); Properties.get().setProperty("lt.lm.insertSentenceTags", "3"); ngrams = provider.getNgrams(seq); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals(PseudoSymbol.SEQUENCE_START.asString(), ngrams[0].get(0)); Assert.assertEquals(PseudoSymbol.SEQUENCE_END.asString(), ngrams[ngrams.length-1].get(provider.getLmOrder()-1)); Properties.get().setProperty("lt.lm.insertSentenceTags", orig); }
@Test public void testSentenceBoundaryHandling() throws Exception { String seq = "The qu1ck br0wn fox jumps over the lazy dog"; StringProviderMXBean provider = AbstractStringProvider.connectToServer(_host, _rmiport, _lm_id); String orig1 = Properties.get().getProperty("lt.lm.insertSentenceTags"); String orig = Properties.get().getProperty("lt.lm.handleBoundaries"); Properties.get().setProperty("lt.lm.insertSentenceTags", "0"); Properties.get().setProperty("lt.lm.handleBoundaries", "-1"); List<String>[] ngrams = provider.getNgrams("The"); System.out.println(Arrays.asList(ngrams)); Assert.assertTrue(ngrams.length == 0); Properties.get().setProperty("lt.lm.handleBoundaries", "0"); ngrams = provider.getNgrams("The"); System.out.println(Arrays.asList(ngrams)); Assert.assertTrue(ngrams.length == 1); Assert.assertEquals("The", ngrams[0].get(0)); Properties.get().setProperty("lt.lm.handleBoundaries", "1"); ngrams = provider.getNgrams(seq); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals("The", ngrams[0].get(0)); Assert.assertEquals("The", ngrams[0].get(1)); Properties.get().setProperty("lt.lm.handleBoundaries", "2"); ngrams = provider.getNgrams(seq); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals("The", ngrams[0].get(0)); Assert.assertEquals("qu1ck", ngrams[0].get(1)); Assert.assertTrue(ngrams[0].size() == 2); Assert.assertTrue(ngrams[1].size() == 3); ngrams = provider.getNgrams("^ hello world"); System.out.println(Arrays.asList(ngrams)); Assert.assertEquals("^", ngrams[0].get(0)); Assert.assertEquals("hello", ngrams[0].get(1)); Assert.assertTrue(ngrams[0].size() == 2); Assert.assertTrue(ngrams[1].size() == 3); Properties.get().setProperty("lt.lm.handleBoundaries", orig); Properties.get().setProperty("lt.lm.insertSentenceTags", orig1); }