@Test public void testParseContraction() throws IOException { InputStreamFactory streamFactory = new ResourceAsStreamFactory(ConlluStreamTest.class, "pt_br-ud-sample.conllu"); try (ObjectStream<TokenSample> stream = new ConlluTokenSampleStream(new ConlluStream(streamFactory))) { TokenSample expected1 = TokenSample.parse( "Numa reunião entre representantes da Secretaria da Criança do DF " + "ea juíza da Vara de Execuções de Medidas Socioeducativas" + TokenSample.DEFAULT_SEPARATOR_CHARS + ", Lavínia Tupi Vieira Fonseca" + TokenSample.DEFAULT_SEPARATOR_CHARS + ", ficou acordado que dos 25 internos" + TokenSample.DEFAULT_SEPARATOR_CHARS + ", 12 serão internados na Unidade de " + "Planaltina e os outros 13 devem retornar para a Unidade do Recanto das Emas" + TokenSample.DEFAULT_SEPARATOR_CHARS + ", antigo Ciago" + TokenSample.DEFAULT_SEPARATOR_CHARS + "." , TokenSample.DEFAULT_SEPARATOR_CHARS); TokenSample predicted = stream.read(); Assert.assertEquals(expected1, predicted); } }
@Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } ConlluTokenSampleStream(ObjectStream<ConlluSentence> samples); }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } ConlluTokenSampleStream(ObjectStream<ConlluSentence> samples); @Override TokenSample read(); }
ConlluTokenSampleStream extends FilterObjectStream<ConlluSentence, TokenSample> { @Override public TokenSample read() throws IOException { ConlluSentence sentence = samples.read(); if (sentence != null) { if (sentence.getTextComment() != null) { StringBuilder text = new StringBuilder(sentence.getTextComment()); int searchIndex = 0; for (ConlluWordLine wordLine : sentence.getWordLines()) { if (wordLine.getId().contains(".")) { continue; } String token = wordLine.getForm(); int tokenIndex = text.indexOf(token, searchIndex); if (tokenIndex == -1) { throw new IOException(String.format("Failed to match token [%s] in sentence [%s] with text [%s]", token, sentence.getSentenceIdComment(), text)); } searchIndex = tokenIndex + token.length(); if (searchIndex < text.length()) { if (!StringUtil.isWhitespace(text.charAt(searchIndex))) { text.insert(searchIndex, TokenSample.DEFAULT_SEPARATOR_CHARS); } } } return TokenSample.parse(text.toString(), TokenSample.DEFAULT_SEPARATOR_CHARS); } else { throw new IOException("Sentence is missing raw text sample!"); } } return null; } ConlluTokenSampleStream(ObjectStream<ConlluSentence> samples); @Override TokenSample read();  }