@Test public void testParsingWhitespaceAndSeparatedString() throws IOException { String sampleTokens = "a b<SPLIT>c d<SPLIT>e"; try (ObjectStream<TokenSample> sampleTokenStream = new TokenSampleStream( ObjectStreamUtils.createObjectStream(sampleTokens))) { TokenSample tokenSample = sampleTokenStream.read(); Span[] tokenSpans = tokenSample.getTokenSpans(); Assert.assertEquals(5, tokenSpans.length); Assert.assertEquals("a", tokenSpans[0].getCoveredText(tokenSample.getText())); Assert.assertEquals("b", tokenSpans[1].getCoveredText(tokenSample.getText())); Assert.assertEquals("c", tokenSpans[2].getCoveredText(tokenSample.getText())); Assert.assertEquals("d", tokenSpans[3].getCoveredText(tokenSample.getText())); Assert.assertEquals("e", tokenSpans[4].getCoveredText(tokenSample.getText())); } }
public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } TokenSampleStream(ObjectStream<String> sampleStrings, String separatorChars);  TokenSampleStream(ObjectStream<String> sentences); }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } TokenSampleStream(ObjectStream<String> sampleStrings, String separatorChars);  TokenSampleStream(ObjectStream<String> sentences); TokenSample read(); }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } TokenSampleStream(ObjectStream<String> sampleStrings, String separatorChars);  TokenSampleStream(ObjectStream<String> sentences); TokenSample read();  }