@Test public void testParseTwoSentences() throws IOException { InputStreamFactory streamFactory = new ResourceAsStreamFactory(ConlluStreamTest.class, "de-ud-train-sample.conllu"); try (ObjectStream<SentenceSample> stream = new ConlluSentenceSampleStream(new ConlluStream(streamFactory), 1)) { SentenceSample sample1 = stream.read(); Assert.assertEquals("Fachlich kompetent, sehr gute Beratung und ein freundliches Team.", sample1.getDocument()); Assert.assertEquals(new Span(0, 65), sample1.getSentences()[0]); SentenceSample sample2 = stream.read(); Assert.assertEquals("Beiden Zahnärzten verdanke ich einen neuen Biss und dadurch " + "endlich keine Rückenschmerzen mehr.", sample2.getDocument()); Assert.assertEquals(new Span(0, 95), sample2.getSentences()[0]); Assert.assertNull("Stream must be exhausted", stream.read()); } try (ObjectStream<SentenceSample> stream = new ConlluSentenceSampleStream(new ConlluStream(streamFactory), 3)) { SentenceSample sample = stream.read(); Assert.assertEquals("Fachlich kompetent, sehr gute Beratung und ein freundliches Team." + " Beiden Zahnärzten verdanke ich einen neuen Biss und dadurch endlich keine " + "Rückenschmerzen mehr.", sample.getDocument()); Assert.assertNull("Stream must be exhausted", stream.read()); } }
@Override public SentenceSample read() throws IOException { StringBuilder documentText = new StringBuilder(); List<Span> sentenceSpans = new ArrayList<>(); ConlluSentence sentence; for (int i = 0; i < sentencesPerSample && (sentence = samples.read()) != null; i++) { int startIndex = documentText.length(); documentText.append(sentence.getTextComment()).append(' '); sentenceSpans.add(new Span(startIndex, documentText.length() - 1)); } if (documentText.length() > 0) { documentText.setLength(documentText.length() - 1); return new SentenceSample(documentText, sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; }
ConlluSentenceSampleStream extends FilterObjectStream<ConlluSentence, SentenceSample> { @Override public SentenceSample read() throws IOException { StringBuilder documentText = new StringBuilder(); List<Span> sentenceSpans = new ArrayList<>(); ConlluSentence sentence; for (int i = 0; i < sentencesPerSample && (sentence = samples.read()) != null; i++) { int startIndex = documentText.length(); documentText.append(sentence.getTextComment()).append(' '); sentenceSpans.add(new Span(startIndex, documentText.length() - 1)); } if (documentText.length() > 0) { documentText.setLength(documentText.length() - 1); return new SentenceSample(documentText, sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } }
ConlluSentenceSampleStream extends FilterObjectStream<ConlluSentence, SentenceSample> { @Override public SentenceSample read() throws IOException { StringBuilder documentText = new StringBuilder(); List<Span> sentenceSpans = new ArrayList<>(); ConlluSentence sentence; for (int i = 0; i < sentencesPerSample && (sentence = samples.read()) != null; i++) { int startIndex = documentText.length(); documentText.append(sentence.getTextComment()).append(' '); sentenceSpans.add(new Span(startIndex, documentText.length() - 1)); } if (documentText.length() > 0) { documentText.setLength(documentText.length() - 1); return new SentenceSample(documentText, sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } ConlluSentenceSampleStream(ObjectStream<ConlluSentence> samples, int sentencesPerSample); }
ConlluSentenceSampleStream extends FilterObjectStream<ConlluSentence, SentenceSample> { @Override public SentenceSample read() throws IOException { StringBuilder documentText = new StringBuilder(); List<Span> sentenceSpans = new ArrayList<>(); ConlluSentence sentence; for (int i = 0; i < sentencesPerSample && (sentence = samples.read()) != null; i++) { int startIndex = documentText.length(); documentText.append(sentence.getTextComment()).append(' '); sentenceSpans.add(new Span(startIndex, documentText.length() - 1)); } if (documentText.length() > 0) { documentText.setLength(documentText.length() - 1); return new SentenceSample(documentText, sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } ConlluSentenceSampleStream(ObjectStream<ConlluSentence> samples, int sentencesPerSample); @Override SentenceSample read(); }
ConlluSentenceSampleStream extends FilterObjectStream<ConlluSentence, SentenceSample> { @Override public SentenceSample read() throws IOException { StringBuilder documentText = new StringBuilder(); List<Span> sentenceSpans = new ArrayList<>(); ConlluSentence sentence; for (int i = 0; i < sentencesPerSample && (sentence = samples.read()) != null; i++) { int startIndex = documentText.length(); documentText.append(sentence.getTextComment()).append(' '); sentenceSpans.add(new Span(startIndex, documentText.length() - 1)); } if (documentText.length() > 0) { documentText.setLength(documentText.length() - 1); return new SentenceSample(documentText, sentenceSpans.toArray(new Span[sentenceSpans.size()])); } return null; } ConlluSentenceSampleStream(ObjectStream<ConlluSentence> samples, int sentencesPerSample); @Override SentenceSample read();  }