@Test public void testParsingWhitespaceSeparatedTokens() throws IOException { String sampleTokens = "Slave to the wage"; ObjectStream<TokenSample> sampleTokenStream = new TokenSampleStream( ObjectStreamUtils.createObjectStream(sampleTokens)); TokenSample tokenSample = sampleTokenStream.read(); Span[] tokenSpans = tokenSample.getTokenSpans(); Assert.assertEquals(4, tokenSpans.length); Assert.assertEquals("Slave", tokenSpans[0].getCoveredText(sampleTokens)); Assert.assertEquals("to", tokenSpans[1].getCoveredText(sampleTokens)); Assert.assertEquals("the", tokenSpans[2].getCoveredText(sampleTokens)); Assert.assertEquals("wage", tokenSpans[3].getCoveredText(sampleTokens)); }
public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } TokenSampleStream(ObjectStream<String> sampleStrings, String separatorChars);  TokenSampleStream(ObjectStream<String> sentences); }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } TokenSampleStream(ObjectStream<String> sampleStrings, String separatorChars);  TokenSampleStream(ObjectStream<String> sentences); TokenSample read(); }
TokenSampleStream extends FilterObjectStream<String, TokenSample> { public TokenSample read() throws IOException { String sampleString = samples.read(); if (sampleString != null) { return TokenSample.parse(sampleString, separatorChars); } else { return null; } } TokenSampleStream(ObjectStream<String> sampleStrings, String separatorChars);  TokenSampleStream(ObjectStream<String> sentences); TokenSample read();  }