@Test public void testParseTwoSentences() throws IOException { InputStreamFactory streamFactory = new ResourceAsStreamFactory(ConlluStreamTest.class, "de-ud-train-sample.conllu"); try (ObjectStream<TokenSample> stream = new ConlluTokenSampleStream(new ConlluStream(streamFactory))) { TokenSample expected1 = TokenSample.parse( "Fachlich kompetent" + TokenSample.DEFAULT_SEPARATOR_CHARS + ", sehr gute Beratung und ein freundliches Team" + TokenSample.DEFAULT_SEPARATOR_CHARS + ".", TokenSample.DEFAULT_SEPARATOR_CHARS); Assert.assertEquals(expected1, stream.read()); TokenSample expected2 = TokenSample.parse("Beiden Zahnärzten verdanke ich einen " + "neuen Biss und dadurch endlich keine Rückenschmerzen mehr" + TokenSample.DEFAULT_SEPARATOR_CHARS + ".", TokenSample.DEFAULT_SEPARATOR_CHARS); Assert.assertEquals(expected2, stream.read()); Assert.assertNull("Stream must be exhausted", stream.read()); } }
@Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } ConlluTokenSampleStream(ObjectStream<ConlluSentence> samples); }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } ConlluTokenSampleStream(ObjectStream<ConlluSentence> samples); @Override TokenSample read(); }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } ConlluTokenSampleStream(ObjectStream<ConlluSentence> samples); @Override TokenSample read();  }