@Test public void testParseTwoSentences() throws IOException { InputStreamFactory streamFactory = new ResourceAsStreamFactory(ConlluStreamTest.class, "de-ud-train-sample.conllu"); try (ObjectStream<ConlluSentence> stream = new ConlluStream(streamFactory)) { ConlluSentence sent1 = stream.read(); Assert.assertEquals("train-s21", sent1.getSentenceIdComment()); Assert.assertEquals("Fachlich kompetent, sehr gute Beratung und ein freundliches Team.", sent1.getTextComment()); Assert.assertEquals(11, sent1.getWordLines().size()); ConlluSentence sent2 = stream.read(); Assert.assertEquals("train-s22", sent2.getSentenceIdComment()); Assert.assertEquals( "Beiden Zahnärzten verdanke ich einen neuen Biss und dadurch endlich keine Rückenschmerzen mehr.", sent2.getTextComment()); Assert.assertEquals(14, sent2.getWordLines().size()); Assert.assertNull("Stream must be exhausted", stream.read()); } }
@Override public ConlluSentence read() throws IOException { String sentence = sentenceStream.read(); if (sentence != null) { List<ConlluWordLine> wordLines = new ArrayList<>(); BufferedReader reader = new BufferedReader(new StringReader(sentence)); String sentenceId = null; String text = null; String line; while ((line = reader.readLine()) != null) { if (line.trim().startsWith("#")) { String commentLine = line.trim().substring(1); int separator = commentLine.indexOf('='); if (separator != -1) { String firstPart = commentLine.substring(0, separator).trim(); String secondPart = commentLine.substring(separator + 1, commentLine.length()).trim(); if (!secondPart.isEmpty()) { switch (firstPart) { case "sent_id": sentenceId = secondPart; break; case "text": text = secondPart; break; } } } } else { wordLines.add(new ConlluWordLine(line)); } } wordLines = postProcessContractions(wordLines); return new ConlluSentence(wordLines, sentenceId, text); } return null; }
ConlluStream implements ObjectStream<ConlluSentence> { @Override public ConlluSentence read() throws IOException { String sentence = sentenceStream.read(); if (sentence != null) { List<ConlluWordLine> wordLines = new ArrayList<>(); BufferedReader reader = new BufferedReader(new StringReader(sentence)); String sentenceId = null; String text = null; String line; while ((line = reader.readLine()) != null) { if (line.trim().startsWith("#")) { String commentLine = line.trim().substring(1); int separator = commentLine.indexOf('='); if (separator != -1) { String firstPart = commentLine.substring(0, separator).trim(); String secondPart = commentLine.substring(separator + 1, commentLine.length()).trim(); if (!secondPart.isEmpty()) { switch (firstPart) { case "sent_id": sentenceId = secondPart; break; case "text": text = secondPart; break; } } } } else { wordLines.add(new ConlluWordLine(line)); } } wordLines = postProcessContractions(wordLines); return new ConlluSentence(wordLines, sentenceId, text); } return null; } }
ConlluStream implements ObjectStream<ConlluSentence> { @Override public ConlluSentence read() throws IOException { String sentence = sentenceStream.read(); if (sentence != null) { List<ConlluWordLine> wordLines = new ArrayList<>(); BufferedReader reader = new BufferedReader(new StringReader(sentence)); String sentenceId = null; String text = null; String line; while ((line = reader.readLine()) != null) { if (line.trim().startsWith("#")) { String commentLine = line.trim().substring(1); int separator = commentLine.indexOf('='); if (separator != -1) { String firstPart = commentLine.substring(0, separator).trim(); String secondPart = commentLine.substring(separator + 1, commentLine.length()).trim(); if (!secondPart.isEmpty()) { switch (firstPart) { case "sent_id": sentenceId = secondPart; break; case "text": text = secondPart; break; } } } } else { wordLines.add(new ConlluWordLine(line)); } } wordLines = postProcessContractions(wordLines); return new ConlluSentence(wordLines, sentenceId, text); } return null; } ConlluStream(InputStreamFactory in); }
ConlluStream implements ObjectStream<ConlluSentence> { @Override public ConlluSentence read() throws IOException { String sentence = sentenceStream.read(); if (sentence != null) { List<ConlluWordLine> wordLines = new ArrayList<>(); BufferedReader reader = new BufferedReader(new StringReader(sentence)); String sentenceId = null; String text = null; String line; while ((line = reader.readLine()) != null) { if (line.trim().startsWith("#")) { String commentLine = line.trim().substring(1); int separator = commentLine.indexOf('='); if (separator != -1) { String firstPart = commentLine.substring(0, separator).trim(); String secondPart = commentLine.substring(separator + 1, commentLine.length()).trim(); if (!secondPart.isEmpty()) { switch (firstPart) { case "sent_id": sentenceId = secondPart; break; case "text": text = secondPart; break; } } } } else { wordLines.add(new ConlluWordLine(line)); } } wordLines = postProcessContractions(wordLines); return new ConlluSentence(wordLines, sentenceId, text); } return null; } ConlluStream(InputStreamFactory in); @Override ConlluSentence read(); @Override void close(); @Override void reset(); }
ConlluStream implements ObjectStream<ConlluSentence> { @Override public ConlluSentence read() throws IOException { String sentence = sentenceStream.read(); if (sentence != null) { List<ConlluWordLine> wordLines = new ArrayList<>(); BufferedReader reader = new BufferedReader(new StringReader(sentence)); String sentenceId = null; String text = null; String line; while ((line = reader.readLine()) != null) { if (line.trim().startsWith("#")) { String commentLine = line.trim().substring(1); int separator = commentLine.indexOf('='); if (separator != -1) { String firstPart = commentLine.substring(0, separator).trim(); String secondPart = commentLine.substring(separator + 1, commentLine.length()).trim(); if (!secondPart.isEmpty()) { switch (firstPart) { case "sent_id": sentenceId = secondPart; break; case "text": text = secondPart; break; } } } } else { wordLines.add(new ConlluWordLine(line)); } } wordLines = postProcessContractions(wordLines); return new ConlluSentence(wordLines, sentenceId, text); } return null; } ConlluStream(InputStreamFactory in); @Override ConlluSentence read(); @Override void close(); @Override void reset();  }